1"python3complete.vim - Omni Completion for python
2" Maintainer: <vacancy>
3" Previous Maintainer: Aaron Griffin <[email protected]>
4" Version: 0.9
5" Last Updated: 2020 Oct 9
6"
7" Roland Puntaier: this file contains adaptations for python3 and is parallel to pythoncomplete.vim
8"
9" Changes
10" TODO:
11" 'info' item output can use some formatting work
12" Add an "unsafe eval" mode, to allow for return type evaluation
13" Complete basic syntax along with import statements
14"   i.e. "import url<c-x,c-o>"
15" Continue parsing on invalid line??
16"
17" v 0.9
18"   * Fixed docstring parsing for classes and functions
19"   * Fixed parsing of *args and **kwargs type arguments
20"   * Better function param parsing to handle things like tuples and
21"     lambda defaults args
22"
23" v 0.8
24"   * Fixed an issue where the FIRST assignment was always used instead of
25"   using a subsequent assignment for a variable
26"   * Fixed a scoping issue when working inside a parameterless function
27"
28"
29" v 0.7
30"   * Fixed function list sorting (_ and __ at the bottom)
31"   * Removed newline removal from docs.  It appears vim handles these better in
32"   recent patches
33"
34" v 0.6:
35"   * Fixed argument completion
36"   * Removed the 'kind' completions, as they are better indicated
37"   with real syntax
38"   * Added tuple assignment parsing (whoops, that was forgotten)
39"   * Fixed import handling when flattening scope
40"
41" v 0.5:
42" Yeah, I skipped a version number - 0.4 was never public.
43"  It was a bugfix version on top of 0.3.  This is a complete
44"  rewrite.
45"
46
47if !has('python3')
48    echo "Error: Required vim compiled with +python3"
49    finish
50endif
51
52function! python3complete#Complete(findstart, base)
53    "findstart = 1 when we need to get the text length
54    if a:findstart == 1
55        let line = getline('.')
56        let idx = col('.')
57        while idx > 0
58            let idx -= 1
59            let c = line[idx]
60            if c =~ '\w'
61                continue
62            elseif ! c =~ '\.'
63                let idx = -1
64                break
65            else
66                break
67            endif
68        endwhile
69
70        return idx
71    "findstart = 0 when we need to return the list of completions
72    else
73        "vim no longer moves the cursor upon completion... fix that
74        let line = getline('.')
75        let idx = col('.')
76        let cword = ''
77        while idx > 0
78            let idx -= 1
79            let c = line[idx]
80            if c =~ '\w' || c =~ '\.'
81                let cword = c . cword
82                continue
83            elseif strlen(cword) > 0 || idx == 0
84                break
85            endif
86        endwhile
87	execute "py3 vimpy3complete('" . escape(cword, "'") . "', '" . escape(a:base, "'") . "')"
88        return g:python3complete_completions
89    endif
90endfunction
91
92function! s:DefPython()
93py3 << PYTHONEOF
94import sys, tokenize, io, types
95from token import NAME, DEDENT, NEWLINE, STRING
96
97debugstmts=[]
98def dbg(s): debugstmts.append(s)
99def showdbg():
100    for d in debugstmts: print("DBG: %s " % d)
101
102def vimpy3complete(context,match):
103    global debugstmts
104    debugstmts = []
105    try:
106        import vim
107        cmpl = Completer()
108        cmpl.evalsource('\n'.join(vim.current.buffer),vim.eval("line('.')"))
109        all = cmpl.get_completions(context,match)
110        all.sort(key=lambda x:x['abbr'].replace('_','z'))
111        dictstr = '['
112        # have to do this for double quoting
113        for cmpl in all:
114            dictstr += '{'
115            for x in cmpl: dictstr += '"%s":"%s",' % (x,cmpl[x])
116            dictstr += '"icase":0},'
117        if dictstr[-1] == ',': dictstr = dictstr[:-1]
118        dictstr += ']'
119        #dbg("dict: %s" % dictstr)
120        vim.command("silent let g:python3complete_completions = %s" % dictstr)
121        #dbg("Completion dict:\n%s" % all)
122    except vim.error:
123        dbg("VIM Error: %s" % vim.error)
124
125class Completer(object):
126    def __init__(self):
127       self.compldict = {}
128       self.parser = PyParser()
129
130    def evalsource(self,text,line=0):
131        sc = self.parser.parse(text,line)
132        src = sc.get_code()
133        dbg("source: %s" % src)
134        try: exec(src,self.compldict)
135        except: dbg("parser: %s, %s" % (sys.exc_info()[0],sys.exc_info()[1]))
136        for l in sc.locals:
137            try: exec(l,self.compldict)
138            except: dbg("locals: %s, %s [%s]" % (sys.exc_info()[0],sys.exc_info()[1],l))
139
140    def _cleanstr(self,doc):
141        return doc.replace('"',' ').replace("'",' ')
142
143    def get_arguments(self,func_obj):
144        def _ctor(class_ob):
145            try: return class_ob.__init__
146            except AttributeError:
147                for base in class_ob.__bases__:
148                    rc = _ctor(base)
149                    if rc is not None: return rc
150            return None
151
152        arg_offset = 1
153        if type(func_obj) == type: func_obj = _ctor(func_obj)
154        elif type(func_obj) == types.MethodType: arg_offset = 1
155        else: arg_offset = 0
156
157        arg_text=''
158        if type(func_obj) in [types.FunctionType, types.LambdaType,types.MethodType]:
159            try:
160                cd = func_obj.__code__
161                real_args = cd.co_varnames[arg_offset:cd.co_argcount]
162                defaults = func_obj.__defaults__ or []
163                defaults = ["=%s" % name for name in defaults]
164                defaults = [""] * (len(real_args)-len(defaults)) + defaults
165                items = [a+d for a,d in zip(real_args,defaults)]
166                if func_obj.__code__.co_flags & 0x4:
167                    items.append("...")
168                if func_obj.__code__.co_flags & 0x8:
169                    items.append("***")
170                arg_text = (','.join(items)) + ')'
171            except:
172                dbg("arg completion: %s: %s" % (sys.exc_info()[0],sys.exc_info()[1]))
173                pass
174        if len(arg_text) == 0:
175            # The doc string sometimes contains the function signature
176            #  this works for a lot of C modules that are part of the
177            #  standard library
178            doc = func_obj.__doc__
179            if doc:
180                doc = doc.lstrip()
181                pos = doc.find('\n')
182                if pos > 0:
183                    sigline = doc[:pos]
184                    lidx = sigline.find('(')
185                    ridx = sigline.find(')')
186                    if lidx > 0 and ridx > 0:
187                        arg_text = sigline[lidx+1:ridx] + ')'
188        if len(arg_text) == 0: arg_text = ')'
189        return arg_text
190
191    def get_completions(self,context,match):
192        #dbg("get_completions('%s','%s')" % (context,match))
193        stmt = ''
194        if context: stmt += str(context)
195        if match: stmt += str(match)
196        try:
197            result = None
198            all = {}
199            ridx = stmt.rfind('.')
200            if len(stmt) > 0 and stmt[-1] == '(':
201                result = eval(_sanitize(stmt[:-1]), self.compldict)
202                doc = result.__doc__
203                if doc is None: doc = ''
204                args = self.get_arguments(result)
205                return [{'word':self._cleanstr(args),'info':self._cleanstr(doc)}]
206            elif ridx == -1:
207                match = stmt
208                all = self.compldict
209            else:
210                match = stmt[ridx+1:]
211                stmt = _sanitize(stmt[:ridx])
212                result = eval(stmt, self.compldict)
213                all = dir(result)
214
215            dbg("completing: stmt:%s" % stmt)
216            completions = []
217
218            try: maindoc = result.__doc__
219            except: maindoc = ' '
220            if maindoc is None: maindoc = ' '
221            for m in all:
222                if m == "_PyCmplNoType": continue #this is internal
223                try:
224                    dbg('possible completion: %s' % m)
225                    if m.find(match) == 0:
226                        if result is None: inst = all[m]
227                        else: inst = getattr(result,m)
228                        try: doc = inst.__doc__
229                        except: doc = maindoc
230                        typestr = str(inst)
231                        if doc is None or doc == '': doc = maindoc
232
233                        wrd = m[len(match):]
234                        c = {'word':wrd, 'abbr':m,  'info':self._cleanstr(doc)}
235                        if "function" in typestr:
236                            c['word'] += '('
237                            c['abbr'] += '(' + self._cleanstr(self.get_arguments(inst))
238                        elif "method" in typestr:
239                            c['word'] += '('
240                            c['abbr'] += '(' + self._cleanstr(self.get_arguments(inst))
241                        elif "module" in typestr:
242                            c['word'] += '.'
243                        elif "type" in typestr:
244                            c['word'] += '('
245                            c['abbr'] += '('
246                        completions.append(c)
247                except:
248                    i = sys.exc_info()
249                    dbg("inner completion: %s,%s [stmt='%s']" % (i[0],i[1],stmt))
250            return completions
251        except:
252            i = sys.exc_info()
253            dbg("completion: %s,%s [stmt='%s']" % (i[0],i[1],stmt))
254            return []
255
256class Scope(object):
257    def __init__(self,name,indent,docstr=''):
258        self.subscopes = []
259        self.docstr = docstr
260        self.locals = []
261        self.parent = None
262        self.name = name
263        self.indent = indent
264
265    def add(self,sub):
266        #print('push scope: [%s@%s]' % (sub.name,sub.indent))
267        sub.parent = self
268        self.subscopes.append(sub)
269        return sub
270
271    def doc(self,str):
272        """ Clean up a docstring """
273        d = str.replace('\n',' ')
274        d = d.replace('\t',' ')
275        while d.find('  ') > -1: d = d.replace('  ',' ')
276        while d[0] in '"\'\t ': d = d[1:]
277        while d[-1] in '"\'\t ': d = d[:-1]
278        dbg("Scope(%s)::docstr = %s" % (self,d))
279        self.docstr = d
280
281    def local(self,loc):
282        self._checkexisting(loc)
283        self.locals.append(loc)
284
285    def copy_decl(self,indent=0):
286        """ Copy a scope's declaration only, at the specified indent level - not local variables """
287        return Scope(self.name,indent,self.docstr)
288
289    def _checkexisting(self,test):
290        "Convienance function... keep out duplicates"
291        if test.find('=') > -1:
292            var = test.split('=')[0].strip()
293            for l in self.locals:
294                if l.find('=') > -1 and var == l.split('=')[0].strip():
295                    self.locals.remove(l)
296
297    def get_code(self):
298        str = ""
299        if len(self.docstr) > 0: str += '"""'+self.docstr+'"""\n'
300        for l in self.locals:
301            if l.startswith('import'): str += l+'\n'
302        str += 'class _PyCmplNoType:\n    def __getattr__(self,name):\n        return None\n'
303        for sub in self.subscopes:
304            str += sub.get_code()
305        for l in self.locals:
306            if not l.startswith('import'): str += l+'\n'
307
308        return str
309
310    def pop(self,indent):
311        #print('pop scope: [%s] to [%s]' % (self.indent,indent))
312        outer = self
313        while outer.parent != None and outer.indent >= indent:
314            outer = outer.parent
315        return outer
316
317    def currentindent(self):
318        #print('parse current indent: %s' % self.indent)
319        return '    '*self.indent
320
321    def childindent(self):
322        #print('parse child indent: [%s]' % (self.indent+1))
323        return '    '*(self.indent+1)
324
325class Class(Scope):
326    def __init__(self, name, supers, indent, docstr=''):
327        Scope.__init__(self,name,indent, docstr)
328        self.supers = supers
329    def copy_decl(self,indent=0):
330        c = Class(self.name,self.supers,indent, self.docstr)
331        for s in self.subscopes:
332            c.add(s.copy_decl(indent+1))
333        return c
334    def get_code(self):
335        str = '%sclass %s' % (self.currentindent(),self.name)
336        if len(self.supers) > 0: str += '(%s)' % ','.join(self.supers)
337        str += ':\n'
338        if len(self.docstr) > 0: str += self.childindent()+'"""'+self.docstr+'"""\n'
339        if len(self.subscopes) > 0:
340            for s in self.subscopes: str += s.get_code()
341        else:
342            str += '%spass\n' % self.childindent()
343        return str
344
345
346class Function(Scope):
347    def __init__(self, name, params, indent, docstr=''):
348        Scope.__init__(self,name,indent, docstr)
349        self.params = params
350    def copy_decl(self,indent=0):
351        return Function(self.name,self.params,indent, self.docstr)
352    def get_code(self):
353        str = "%sdef %s(%s):\n" % \
354            (self.currentindent(),self.name,','.join(self.params))
355        if len(self.docstr) > 0: str += self.childindent()+'"""'+self.docstr+'"""\n'
356        str += "%spass\n" % self.childindent()
357        return str
358
359class PyParser:
360    def __init__(self):
361        self.top = Scope('global',0)
362        self.scope = self.top
363        self.parserline = 0
364
365    def _parsedotname(self,pre=None):
366        #returns (dottedname, nexttoken)
367        name = []
368        if pre is None:
369            tokentype, token, indent = self.donext()
370            if tokentype != NAME and token != '*':
371                return ('', token)
372        else: token = pre
373        name.append(token)
374        while True:
375            tokentype, token, indent = self.donext()
376            if token != '.': break
377            tokentype, token, indent = self.donext()
378            if tokentype != NAME: break
379            name.append(token)
380        return (".".join(name), token)
381
382    def _parseimportlist(self):
383        imports = []
384        while True:
385            name, token = self._parsedotname()
386            if not name: break
387            name2 = ''
388            if token == 'as': name2, token = self._parsedotname()
389            imports.append((name, name2))
390            while token != "," and "\n" not in token:
391                tokentype, token, indent = self.donext()
392            if token != ",": break
393        return imports
394
395    def _parenparse(self):
396        name = ''
397        names = []
398        level = 1
399        while True:
400            tokentype, token, indent = self.donext()
401            if token in (')', ',') and level == 1:
402                if '=' not in name: name = name.replace(' ', '')
403                names.append(name.strip())
404                name = ''
405            if token == '(':
406                level += 1
407                name += "("
408            elif token == ')':
409                level -= 1
410                if level == 0: break
411                else: name += ")"
412            elif token == ',' and level == 1:
413                pass
414            else:
415                name += "%s " % str(token)
416        return names
417
418    def _parsefunction(self,indent):
419        self.scope=self.scope.pop(indent)
420        tokentype, fname, ind = self.donext()
421        if tokentype != NAME: return None
422
423        tokentype, open, ind = self.donext()
424        if open != '(': return None
425        params=self._parenparse()
426
427        tokentype, colon, ind = self.donext()
428        if colon != ':': return None
429
430        return Function(fname,params,indent)
431
432    def _parseclass(self,indent):
433        self.scope=self.scope.pop(indent)
434        tokentype, cname, ind = self.donext()
435        if tokentype != NAME: return None
436
437        super = []
438        tokentype, thenext, ind = self.donext()
439        if thenext == '(':
440            super=self._parenparse()
441        elif thenext != ':': return None
442
443        return Class(cname,super,indent)
444
445    def _parseassignment(self):
446        assign=''
447        tokentype, token, indent = self.donext()
448        if tokentype == tokenize.STRING or token == 'str':
449            return '""'
450        elif token == '(' or token == 'tuple':
451            return '()'
452        elif token == '[' or token == 'list':
453            return '[]'
454        elif token == '{' or token == 'dict':
455            return '{}'
456        elif tokentype == tokenize.NUMBER:
457            return '0'
458        elif token == 'open' or token == 'file':
459            return 'file'
460        elif token == 'None':
461            return '_PyCmplNoType()'
462        elif token == 'type':
463            return 'type(_PyCmplNoType)' #only for method resolution
464        else:
465            assign += token
466            level = 0
467            while True:
468                tokentype, token, indent = self.donext()
469                if token in ('(','{','['):
470                    level += 1
471                elif token in (']','}',')'):
472                    level -= 1
473                    if level == 0: break
474                elif level == 0:
475                    if token in (';','\n'): break
476                    assign += token
477        return "%s" % assign
478
479    def donext(self):
480        type, token, (lineno, indent), end, self.parserline = next(self.gen)
481        if lineno == self.curline:
482            #print('line found [%s] scope=%s' % (line.replace('\n',''),self.scope.name))
483            self.currentscope = self.scope
484        return (type, token, indent)
485
486    def _adjustvisibility(self):
487        newscope = Scope('result',0)
488        scp = self.currentscope
489        while scp != None:
490            if type(scp) == Function:
491                slice = 0
492                #Handle 'self' params
493                if scp.parent != None and type(scp.parent) == Class:
494                    slice = 1
495                    newscope.local('%s = %s' % (scp.params[0],scp.parent.name))
496                for p in scp.params[slice:]:
497                    i = p.find('=')
498                    if len(p) == 0: continue
499                    pvar = ''
500                    ptype = ''
501                    if i == -1:
502                        pvar = p
503                        ptype = '_PyCmplNoType()'
504                    else:
505                        pvar = p[:i]
506                        ptype = _sanitize(p[i+1:])
507                    if pvar.startswith('**'):
508                        pvar = pvar[2:]
509                        ptype = '{}'
510                    elif pvar.startswith('*'):
511                        pvar = pvar[1:]
512                        ptype = '[]'
513
514                    newscope.local('%s = %s' % (pvar,ptype))
515
516            for s in scp.subscopes:
517                ns = s.copy_decl(0)
518                newscope.add(ns)
519            for l in scp.locals: newscope.local(l)
520            scp = scp.parent
521
522        self.currentscope = newscope
523        return self.currentscope
524
525    #p.parse(vim.current.buffer[:],vim.eval("line('.')"))
526    def parse(self,text,curline=0):
527        self.curline = int(curline)
528        buf = io.StringIO(''.join(text) + '\n')
529        self.gen = tokenize.generate_tokens(buf.readline)
530        self.currentscope = self.scope
531
532        try:
533            freshscope=True
534            while True:
535                tokentype, token, indent = self.donext()
536                #dbg( 'main: token=[%s] indent=[%s]' % (token,indent))
537
538                if tokentype == DEDENT or token == "pass":
539                    self.scope = self.scope.pop(indent)
540                elif token == 'def':
541                    func = self._parsefunction(indent)
542                    if func is None:
543                        print("function: syntax error...")
544                        continue
545                    dbg("new scope: function")
546                    freshscope = True
547                    self.scope = self.scope.add(func)
548                elif token == 'class':
549                    cls = self._parseclass(indent)
550                    if cls is None:
551                        print("class: syntax error...")
552                        continue
553                    freshscope = True
554                    dbg("new scope: class")
555                    self.scope = self.scope.add(cls)
556
557                elif token == 'import':
558                    imports = self._parseimportlist()
559                    for mod, alias in imports:
560                        loc = "import %s" % mod
561                        if len(alias) > 0: loc += " as %s" % alias
562                        self.scope.local(loc)
563                    freshscope = False
564                elif token == 'from':
565                    mod, token = self._parsedotname()
566                    if not mod or token != "import":
567                        print("from: syntax error...")
568                        continue
569                    names = self._parseimportlist()
570                    for name, alias in names:
571                        loc = "from %s import %s" % (mod,name)
572                        if len(alias) > 0: loc += " as %s" % alias
573                        self.scope.local(loc)
574                    freshscope = False
575                elif tokentype == STRING:
576                    if freshscope: self.scope.doc(token)
577                elif tokentype == NAME:
578                    name,token = self._parsedotname(token)
579                    if token == '=':
580                        stmt = self._parseassignment()
581                        dbg("parseassignment: %s = %s" % (name, stmt))
582                        if stmt != None:
583                            self.scope.local("%s = %s" % (name,stmt))
584                    freshscope = False
585        except StopIteration: #thrown on EOF
586            pass
587        except:
588            dbg("parse error: %s, %s @ %s" %
589                (sys.exc_info()[0], sys.exc_info()[1], self.parserline))
590        return self._adjustvisibility()
591
592def _sanitize(str):
593    val = ''
594    level = 0
595    for c in str:
596        if c in ('(','{','['):
597            level += 1
598        elif c in (']','}',')'):
599            level -= 1
600        elif level == 0:
601            val += c
602    return val
603
604sys.path.extend(['.','..'])
605PYTHONEOF
606endfunction
607
608call s:DefPython()
609