Clean up type system

src/Analysis/Engine/Impl/AnalysisValueSetExtensions.cs

@@ -218,7 +218,7 @@ internal static AnalysisValue GetUnionType(this IAnalysisSet types) {
         /// Gets instance representations of all members of the set.
         /// </summary>
         public static IAnalysisSet GetInstanceType(this IAnalysisSet types)
-            => AnalysisSet.Create(types.SelectMany(ns => (ns.PythonType as IPythonType2)?.IsClass == true ? ns : ns.GetInstanceType()));
+            => AnalysisSet.Create(types.SelectMany(ns => ns.PythonType?.IsTypeFactory == true ? ns : ns.GetInstanceType()));
 
         public static bool IsUnknown(this IAnalysisSet res) {
             return res == null ||

src/Analysis/Engine/Impl/Analyzer/DDG.cs

@@ -9,7 +9,7 @@
 // THIS CODE IS PROVIDED ON AN  *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
 // OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
 // IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
-// MERCHANTABLITY OR NON-INFRINGEMENT.
+// MERCHANTABILITY OR NON-INFRINGEMENT.
 //
 // See the Apache Version 2.0 License for specific language governing
 // permissions and limitations under the License.

src/Analysis/Engine/Impl/Definitions/IOverloadResult.cs

@@ -9,21 +9,40 @@
 // THIS CODE IS PROVIDED ON AN  *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
 // OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
 // IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
-// MERCHANTABLITY OR NON-INFRINGEMENT.
+// MERCHANTABILITY OR NON-INFRINGEMENT.
 //
 // See the Apache Version 2.0 License for specific language governing
 // permissions and limitations under the License.
 
 using System.Collections.Generic;
-using System.Diagnostics.CodeAnalysis;
 
 namespace Microsoft.PythonTools.Analysis {
     public interface IOverloadResult {
+        /// <summary>
+        /// Function name.
+        /// </summary>
         string Name { get; }
+
+        /// <summary>
+        /// Function documentation.
+        /// </summary>
         string Documentation { get; }
-        [SuppressMessage("Microsoft.Performance", "CA1819:PropertiesShouldNotReturnArrays",
-            Justification = "breaking change")]
+
+        /// <summary>
+        /// First parameter if removed from the set.
+        /// Typically 'self' or 'cls'.
+        /// </summary>
+        ParameterResult FirstParameter { get; }
+
+        /// <summary>
+        /// Function parameters. First parameter may be removed, in which case
+        /// it is present as <see cref="FirstParameter"/>.
+        /// </summary>
         ParameterResult[] Parameters { get; }
+
+        /// <summary>
+        /// Possible return types.
+        /// </summary>
         IReadOnlyList<string> ReturnType { get; }
     }
 }

src/Analysis/Engine/Impl/EmptyBuiltinModule.cs

@@ -16,14 +16,11 @@
 
 using System.Collections.Generic;
 using Microsoft.PythonTools.Interpreter;
+using Microsoft.PythonTools.Interpreter.Ast;
 
 namespace Microsoft.PythonTools.Analysis {
-    class EmptyBuiltinModule : IBuiltinPythonModule {
-        private readonly string _name;
-
-        public EmptyBuiltinModule(string name) {
-            _name = name;
-        }
+    class EmptyBuiltinModule : PythonModuleType, IBuiltinPythonModule {
+        public EmptyBuiltinModule(string name): base(name) { }
 
         #region IBuiltinPythonModule Members
 
@@ -35,42 +32,12 @@ public IMember GetAnyMember(string name) {
 
         #region IPythonModule Members
 
-        public string Name {
-            get { return _name; }
-        }
-
         public IEnumerable<string> GetChildrenModules() {
             yield break;
         }
 
         public void Imported(IModuleContext context) {
         }
-
-        public string Documentation {
-            get { return string.Empty; }
-        }
-
-        #endregion
-
-        #region IMemberContainer Members
-
-        public IMember GetMember(IModuleContext context, string name) {
-            return null;
-        }
-
-        public IEnumerable<string> GetMemberNames(IModuleContext moduleContext) {
-            yield break;
-        }
-
         #endregion
-
-        #region IMember Members
-
-        public PythonMemberType MemberType {
-            get { return PythonMemberType.Module; }
-        }
-
-        #endregion
-
     }
 }

src/Analysis/Engine/Impl/Infrastructure/Extensions/EnumerableExtensions.cs

@@ -23,31 +23,21 @@ static class EnumerableExtensions {
         public static bool IsNullOrEmpty<T>(this IEnumerable<T> source)
             => source == null || !source.Any();
 
-        public static T[] MaybeEnumerate<T>(this T[] source) {
-            return source ?? Array.Empty<T>();
-        }
+        public static T[] MaybeEnumerate<T>(this T[] source) => source ?? Array.Empty<T>();
 
-        public static IEnumerable<T> MaybeEnumerate<T>(this IEnumerable<T> source) {
-            return source ?? Enumerable.Empty<T>();
-        }
+        public static IEnumerable<T> MaybeEnumerate<T>(this IEnumerable<T> source) => source ?? Enumerable.Empty<T>();
 
-        private static T Identity<T>(T source) {
-            return source;
-        }
+        private static T Identity<T>(T source) => source;
 
-        public static IEnumerable<T> SelectMany<T>(this IEnumerable<IEnumerable<T>> source) {
-            return source.SelectMany(Identity);
-        }
+        public static IEnumerable<T> SelectMany<T>(this IEnumerable<IEnumerable<T>> source) => source.SelectMany(Identity);
 
-        public static IEnumerable<T> Ordered<T>(this IEnumerable<T> source) {
-            return source.OrderBy(Identity);
-        }
+        public static IEnumerable<T> Ordered<T>(this IEnumerable<T> source)
+            => source.OrderBy(Identity);
 
         private static bool NotNull<T>(T obj) where T : class => obj != null;
 
-        public static IEnumerable<T> WhereNotNull<T>(this IEnumerable<T> source) where T : class {
-            return source.Where(NotNull);
-        }
+        public static IEnumerable<T> WhereNotNull<T>(this IEnumerable<T> source) where T : class
+            => source != null ? source.Where(NotNull) : Enumerable.Empty<T>();
 
         public static bool SetEquals<T>(this IEnumerable<T> source, IEnumerable<T> other,
             IEqualityComparer<T> comparer = null) where T : class {

src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisFunctionWalker.cs

@@ -65,7 +65,9 @@ public void Walk() {
             var self = GetSelf();
             _selfType = (self as AstPythonConstant)?.Type as AstPythonType;
 
-            _overload.ReturnTypes.AddRange(_scope.GetTypesFromAnnotation(Target.ReturnAnnotation).ExcludeDefault());
+            var annotationTypes = _scope.GetTypesFromAnnotation(Target.ReturnAnnotation).ExcludeDefault();
+            _overload.ReturnTypes.AddRange(annotationTypes);
+
             _scope.PushScope();
 
             // Declare self, if any
@@ -84,7 +86,10 @@ public void Walk() {
                 _scope.SetInScope(p.Name, value ?? _scope.UnknownType);
             }
 
-            Target.Walk(this);
+            // return type from the annotation always wins, no need to walk the body.
+            if (!annotationTypes.Any()) {
+                Target.Walk(this);
+            }
             _scope.PopScope();
         }
 
@@ -165,7 +170,8 @@ public override bool Walk(IfStatement node) {
                 if (name != null && typeName != null) {
                     var typeId = typeName.GetTypeId();
                     if (typeId != BuiltinTypeId.Unknown) {
-                        _scope.SetInScope(name, new AstPythonConstant(new AstPythonBuiltinType(typeName, typeId)));
+                        _scope.SetInScope(name, 
+                            new AstPythonConstant(new AstPythonType(typeName, typeId)));
                     }
                 }
             }

src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisWalker.cs

@@ -115,7 +115,7 @@ internal LocationInfo GetLoc(ClassDefinition node) {
         private LocationInfo GetLoc(Node node) => _scope.GetLoc(node);
 
         private IMember Clone(IMember member) =>
-            member is IPythonMultipleMembers mm ? AstPythonMultipleMembers.Create(mm.Members) :
+            member is IPythonMultipleMembers mm ? AstPythonMultipleMembers.Create(mm.GetMembers()) :
             member;
 
         public override bool Walk(AssignmentStatement node) {
@@ -423,14 +423,18 @@ public override bool Walk(FunctionDefinition node) {
             var dec = (node.Decorators?.Decorators).MaybeEnumerate().ExcludeDefault();
             foreach (var d in dec) {
                 var obj = _scope.GetValueFromExpression(d);
+
+                var declaringType = obj as IPythonType;
+                var declaringModule = declaringType?.DeclaringModule;
+
                 if (obj == _interpreter.GetBuiltinType(BuiltinTypeId.Property)) {
-                    AddProperty(node);
+                    AddProperty(node, declaringModule, declaringType);
                     return false;
                 }
-                var mod = (obj as IPythonType)?.DeclaringModule ?? (obj as IPythonFunction)?.DeclaringModule;
-                var name = (obj as IPythonType)?.Name ?? (obj as IPythonFunction)?.Name;
-                if (mod?.Name == "abc" && name == "abstractproperty") {
-                    AddProperty(node);
+
+                var name = declaringType?.Name;
+                if (declaringModule?.Name == "abc" && name == "abstractproperty") {
+                    AddProperty(node, declaringModule, declaringType);
                     return false;
                 }
             }
@@ -450,10 +454,10 @@ public override bool Walk(FunctionDefinition node) {
             return false;
         }
 
-        private void AddProperty(FunctionDefinition node) {
+        private void AddProperty(FunctionDefinition node, IPythonModule declaringModule, IPythonType declaringType) {
             var existing = _scope.LookupNameInScopes(node.Name, NameLookupContext.LookupOptions.Local) as AstPythonProperty;
             if (existing == null) {
-                existing = new AstPythonProperty(_ast, node, GetLoc(node));
+                existing = new AstPythonProperty(node, declaringModule, declaringType, GetLoc(node));
                 _scope.SetInScope(node.Name, existing);
             }
 
@@ -471,8 +475,8 @@ private IPythonFunctionOverload CreateFunctionOverload(NameLookupContext funcSco
                 .ToArray();
 
             var overload = new AstPythonFunctionOverload(
-                parameters, 
-                funcScope.GetLocOfName(node, node.NameExpression), 
+                parameters,
+                funcScope.GetLocOfName(node, node.NameExpression),
                 node.ReturnAnnotation?.ToCodeString(_ast));
             _functionWalkers.Add(new AstAnalysisFunctionWalker(funcScope, node, overload));
 
@@ -485,14 +489,13 @@ private static string GetDoc(SuiteStatement node) {
             return ce?.Value as string;
         }
 
-        private AstPythonType CreateType(ClassDefinition node) {
-            if (node == null) {
-                throw new ArgumentNullException(nameof(node));
-            }
-            return CreateBuiltinTypes
-                ? new AstPythonBuiltinType(node.Name, _module, node.StartIndex, GetDoc(node.Body as SuiteStatement), GetLoc(node))
-                : new AstPythonType(node.Name,  _module, node.StartIndex, GetDoc(node.Body as SuiteStatement), GetLoc(node));
+        private AstPythonClass CreateClass(ClassDefinition node) {
+            node = node ?? throw new ArgumentNullException(nameof(node));
+            return new AstPythonClass(node, _module,
+                GetDoc(node.Body as SuiteStatement), GetLoc(node),
+                CreateBuiltinTypes ? BuiltinTypeId.Unknown : BuiltinTypeId.Type); // built-ins set type later
         }
+
         private void CollectTopLevelDefinitions() {
             var s = (_ast.Body as SuiteStatement).Statements.ToArray();
 
@@ -501,7 +504,7 @@ private void CollectTopLevelDefinitions() {
             }
 
             foreach (var node in s.OfType<ClassDefinition>()) {
-                _members[node.Name] = CreateType(node);
+                _members[node.Name] = CreateClass(node);
             }
 
             foreach (var node in s.OfType<AssignmentStatement>().Where(n => n.Right is NameExpression)) {
@@ -516,10 +519,12 @@ private void CollectTopLevelDefinitions() {
 
         public override bool Walk(ClassDefinition node) {
             var member = _scope.GetInScope(node.Name);
-            var t = member as AstPythonType ??
-                (member as IPythonMultipleMembers)?.Members.OfType<AstPythonType>().FirstOrDefault(pt => pt.StartIndex == node.StartIndex);
+            var t = member as AstPythonClass;
+            if (t == null && member is IPythonMultipleMembers mm) {
+                t = mm.GetMembers().OfType<AstPythonClass>().FirstOrDefault(pt => pt.ClassDefinition.StartIndex == node.StartIndex);
+            }
             if (t == null) {
-                t = CreateType(node);
+                t = CreateClass(node);
                 _scope.SetInScope(node.Name, t);
             }
 
@@ -549,7 +554,7 @@ public void ProcessFunctionDefinition(FunctionDefinition node) {
             var existing = _scope.LookupNameInScopes(node.Name, NameLookupContext.LookupOptions.Local) as AstPythonFunction;
             if (existing == null) {
                 var cls = _scope.GetInScope("__class__") as IPythonType;
-                existing = new AstPythonFunction(_ast, _module, cls, node, GetLoc(node));
+                existing = new AstPythonFunction(node, _module, cls, GetLoc(node));
                 _scope.SetInScope(node.Name, existing);
             }
 

src/Analysis/Engine/Impl/Interpreter/Ast/AstBuiltinsPythonModule.cs

@@ -9,7 +9,7 @@
 // THIS CODE IS PROVIDED ON AN  *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
 // OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
 // IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
-// MERCHANTABLITY OR NON-INFRINGEMENT.
+// MERCHANTABILITY OR NON-INFRINGEMENT.
 //
 // See the Apache Version 2.0 License for specific language governing
 // permissions and limitations under the License.
@@ -43,8 +43,7 @@ public override IMember GetMember(IModuleContext context, string name) {
 
         public IMember GetAnyMember(string name) {
             lock (_members) {
-                IMember m;
-                _members.TryGetValue(name, out m);
+                _members.TryGetValue(name, out var m);
                 return m;
             }
         }
@@ -96,13 +95,12 @@ protected override PythonWalker PrepareWalker(AstPythonInterpreter interpreter,
         }
 
         protected override void PostWalk(PythonWalker walker) {
-            AstPythonBuiltinType boolType = null;
-            AstPythonBuiltinType noneType = null;
+            IPythonType boolType = null;
+            IPythonType noneType = null;
 
             foreach (BuiltinTypeId typeId in Enum.GetValues(typeof(BuiltinTypeId))) {
-                if (_members.TryGetValue("__{0}__".FormatInvariant(typeId), out IMember m) && m is AstPythonBuiltinType biType) {
-                    if (typeId != BuiltinTypeId.Str &&
-                        typeId != BuiltinTypeId.StrIterator) {
+                if (_members.TryGetValue("__{0}__".FormatInvariant(typeId), out var m) && m is AstPythonType biType && biType.IsBuiltin) {
+                    if (typeId != BuiltinTypeId.Str && typeId != BuiltinTypeId.StrIterator) {
                         biType.TrySetTypeId(typeId);
                     }
 

src/Analysis/Engine/Impl/Interpreter/Ast/AstNestedPythonModule.cs

@@ -22,19 +22,15 @@
 using Microsoft.PythonTools.Analysis.Infrastructure;
 
 namespace Microsoft.PythonTools.Interpreter.Ast {
-    internal sealed class AstNestedPythonModule : IPythonModule, ILocatedMember {
-        private readonly string _name;
+    internal sealed class AstNestedPythonModule : PythonModuleType, IPythonModule, ILocatedMember {
         private readonly IPythonInterpreter _interpreter;
         private IPythonModule _module;
 
-        public AstNestedPythonModule(IPythonInterpreter interpreter, string fullName) {
+        public AstNestedPythonModule(IPythonInterpreter interpreter, string fullName) : base(fullName) {
             _interpreter = interpreter ?? throw new ArgumentNullException(nameof(interpreter));
-            _name = fullName ?? throw new ArgumentNullException(nameof(fullName));
         }
 
-        public string Name => MaybeModule?.Name ?? _name;
-        public string Documentation => MaybeModule?.Documentation ?? string.Empty;
-        public PythonMemberType MemberType => PythonMemberType.Module;
+        public override string Documentation => MaybeModule?.Documentation ?? string.Empty;
         public IEnumerable<ILocationInfo> Locations => ((MaybeModule as ILocatedMember)?.Locations).MaybeEnumerate();
 
         public bool IsLoaded => MaybeModule != null;
@@ -46,23 +42,24 @@ private IPythonModule GetModule() {
                 return module;
             }
 
-            module = _interpreter.ImportModule(_name);
+            module = _interpreter.ImportModule(Name);
             if (module != null) {
                 Debug.Assert(!(module is AstNestedPythonModule), "ImportModule should not return nested module");
             }
 
             if (module == null) {
-                module = new SentinelModule(_name, false);
+                module = new SentinelModule(Name, false);
             }
 
             return Interlocked.CompareExchange(ref _module, module, null) ?? module;
         }
 
         public IEnumerable<string> GetChildrenModules() => GetModule().GetChildrenModules();
 
-        public IMember GetMember(IModuleContext context, string name) => GetModule().GetMember(context, name);
+        public override IMember GetMember(IModuleContext context, string name)
+            => GetModule().GetMember(context, name);
 
-        public IEnumerable<string> GetMemberNames(IModuleContext context) =>
+        public override IEnumerable<string> GetMemberNames(IModuleContext context) =>
             // TODO: Make GetMemberNames() faster than Imported()
             GetModule().GetMemberNames(context);