Github Copilot

Github CopilotGitHubOpenAICopilot

VSCode

Copilot

VSCodeextension.js

mac~/.vscode

1. `webpack_modules`

webpackjsbundle

AST

babel-parserAST

const ast = parser.parse(source);

babel-traverseASTmodules

function parseModules() {
  traverse(ast, {
    enter(path) {
      if (
        path.node.type === "VariableDeclarator" &&
        path.node.id.name === "__webpack_modules__"
      ) {
        const modules = path.node.init.properties;
        for (const module of modules) {
          const moduleId = module.key.value;
          const moduleAst = module.value;
          const moduleSource = generate(moduleAst).code;

          try {
            const ast = transformRequire(prettier(clearfyParams(moduleId, moduleSource)));

            const mainBody = ast.program.body[0].expression.body.body;
            const moduleCode = generate(types.Program(mainBody)).code;
            fs.writeFileSync(
              "./prettier/modules/" + moduleId + ".js",
              moduleCode,
              "utf8"
            );
          } catch (e) {
            console.log(e);
          }
        }
      }
    },
  });
}

astbabel-generatorbabel-typesast

idbundlecopilotbundle752

2.

bundle

webpackmodule``exports``require

function clearfyParams(moduleId, moduleSource) {
  if (moduleSource.trim().startsWith("function")) {
    // change `function(e, t, n) {` to `(e, t, n) => {`
    moduleSource = moduleSource.replace("function", "");
    moduleSource = moduleSource.replace(")", ") =>");
  }

  const moduleAst = parser.parse(moduleSource);
  let flag = false;

  traverse(moduleAst, {
    ArrowFunctionExpression(path) {
      if (flag) return;
      const params = path.node.params;
      params.forEach((param) => {
        if (param.name === "e" || param.name === "t" || param.name === "n") {
          path.scope.rename(
            param.name,
            {
              e: "module",
              t: "exports",
              n: "require",
            }[param.name]
          );
        }
      });
      flag = true;
    },
  });
  return moduleAst;
}

requireexports

var r = require(12781).Stream;
var i = require(73837);
function o() {
  this.source = null;
  this.dataSize = 0;
  this.maxDataSize = 1048576;
  this.pauseStream = !0;
  this._maxDataSizeExceeded = !1;
  this._released = !1;
  this._bufferedEvents = [];
}
module.exports = o;

3.

JShttps://github.com/thakkarparth007/copilot-explorer

function prettier(ast) {
  const moduleTransformer = {
    // e.g., `(0, r.getConfig)(e, r.ConfigKey.DebugOverrideProxyUrl);`
    // gets transformed to r.getConfig(e, r.ConfigKey.DebugOverrideProxyUrl);
    CallExpression(path) {
      if (path.node.callee.type != "SequenceExpression") {
        return;
      }
      if (
        path.node.callee.expressions.length == 2 &&
        path.node.callee.expressions[0].type == "NumericLiteral"
      ) {
        path.node.callee = path.node.callee.expressions[1];
      }
    },
    ExpressionStatement(path) {
      if (path.node.expression.type == "SequenceExpression") {
        const exprs = path.node.expression.expressions;
        let exprStmts = exprs.map((e) => {
          return types.expressionStatement(e);
        });
        path.replaceWithMultiple(exprStmts);
        return;
      }
      if (path.node.expression.type == "AssignmentExpression") {
        // handle cases like: `a = (expr1, expr2, expr3)`
        // convert to: `expr1; expr2; a = expr3;`
        if (path.node.expression.right.type == "SequenceExpression") {
          const exprs = path.node.expression.right.expressions;
          let exprStmts = exprs.map((e) => {
            return types.expressionStatement(e);
          });
          let lastExpr = exprStmts.pop();
          path.node.expression.right = lastExpr.expression;
          exprStmts.push(path.node);
          path.replaceWithMultiple(exprStmts);
          return;
        }

        // handle cases like: `exports.GoodExplainableName = a;` where `a` is a function or a class
        // rename `a` to `GoodExplainableName` everywhere in the module
        if (
          path.node.expression.left.type == "MemberExpression" &&
          path.node.expression.left.object.type == "Identifier" &&
          path.node.expression.left.object.name == "exports" &&
          path.node.expression.left.property.type == "Identifier" &&
          path.node.expression.left.property.name != "default" &&
          path.node.expression.right.type == "Identifier" &&
          path.node.expression.right.name.length == 1
        ) {
          path.scope.rename(
            path.node.expression.right.name,
            path.node.expression.left.property.name
          );
          return;
        }
      }
      if (path.node.expression.type == "ConditionalExpression") {
        // handle cases like: `<test> ? c : d;`
        // convert to: `if (<test>) { c; } else { d; }`
        const test = path.node.expression.test;
        const consequent = path.node.expression.consequent;
        const alternate = path.node.expression.alternate;

        const ifStmt = types.ifStatement(
          test,
          types.blockStatement([types.expressionStatement(consequent)]),
          types.blockStatement([types.expressionStatement(alternate)])
        );
        path.replaceWith(ifStmt);
        return;
      }
      if (path.node.expression.type == "LogicalExpression") {
        // handle cases like: `a && b;`
        // convert to: `if (a) { b; }`
        const test = path.node.expression.left;
        const consequent = path.node.expression.right;

        const ifStmt = types.ifStatement(
          test,
          types.blockStatement([types.expressionStatement(consequent)]),
          null
        );
        path.replaceWith(ifStmt);
        return;
      }
    },
    IfStatement(path) {
      if (!path.node.test || path.node.test.type != "SequenceExpression") {
        return;
      }
      const exprs = path.node.test.expressions;
      let exprStmts = exprs.map((e) => {
        return types.expressionStatement(e);
      });
      let lastExpr = exprStmts.pop();
      path.node.test = lastExpr.expression;
      exprStmts.push(path.node);
      path.replaceWithMultiple(exprStmts);
    },
    ReturnStatement(path) {
      if (
        !path.node.argument ||
        path.node.argument.type != "SequenceExpression"
      ) {
        return;
      }
      const exprs = path.node.argument.expressions;
      let exprStmts = exprs.map((e) => {
        return types.expressionStatement(e);
      });
      let lastExpr = exprStmts.pop();
      let returnStmt = types.returnStatement(lastExpr.expression);
      exprStmts.push(returnStmt);
      path.replaceWithMultiple(exprStmts);
    },
    VariableDeclaration(path) {
      // change `const a = 1, b = 2;` to `const a = 1; const b = 2;`
      if (path.node.declarations.length > 1) {
        let newDecls = path.node.declarations.map((d) => {
          return types.variableDeclaration(path.node.kind, [d]);
        });
        path.replaceWithMultiple(newDecls);
      }
    },
  };
  traverse(ast, moduleTransformer);
  return ast;
}

4. requireid

requiremoduleidGPTmapastid

function transformRequire(ast) {
  const moduleTransformer = {
    VariableDeclaration(path) {
        if (path.node.declarations[0].init && path.node.declarations[0].init.type === "CallExpression") {
            if (path.node.declarations[0].init.callee.name === "require") {
                const moduleId = path.node.declarations[0].init.arguments[0].value;
                if (NameMap[moduleId]) {
                    const { name, path: modulePath} = NameMap[moduleId];

                    path.node.declarations[0].init.arguments[0].value = '"'+modulePath+'"';
                    path.scope.rename(path.node.declarations[0].id.name, name);
                }
              }
        }
      
    },
  };
  traverse(ast, moduleTransformer);
  return ast;
}

id91238

VSCodeactivecopilotcontextcontext

copilot

registerGhostText

vscodeInlineCompletionItemProvider

InlineSuggestEnabledocumentreturn
getGhostTexttexts
completionsFromGhostTextResultscompletionsTab

getGhostText

1. Prompt

const prompt = await extractprompt.extractPrompt(ctx, document, position);

prompt

2.

if ("copilotNotAvailable" === prompt.type) {
    exports.ghostTextLogger.debug(
      ctx,
      "Copilot not available, due to the .copilotignore settings"
    );
    return {
      type: "abortedBeforeIssued",
      reason: "Copilot not available due to the .copilotignore settings",
    };
  }
  if ("contextTooShort" === prompt.type) {
    exports.ghostTextLogger.debug(ctx, "Breaking, not enough context");
    return {
      type: "abortedBeforeIssued",
      reason: "Not enough context",
    };
  }
  if (token?.isCancellationRequested) {
    exports.ghostTextLogger.info(ctx, "Cancelled after extractPrompt");
    return {
      type: "abortedBeforeIssued",
      reason: "Cancelled after extractPrompt",
    };
  }

.copilotignore

3.

copilotprefix``suffix

function updateGlobalCacheKey(prefix, suffix, promptKey) {
  prefixCache = prefix;
  suffixCache = suffix;
  promptKeyCache = promptKey;
}

promptKey``prefix``suffixcopilotprefixsuffix

copilotprompt

copilotLRUprompt100

exports.completionCache = new s.LRUCacheMap(100);

keyForPromptprefixsuffixhash

exports.keyForPrompt = function (e) {
  return r.SHA256(e.prefix + e.suffix).toString();
};

4.

promptcopilot

Debounce
contexualFilterScore

copilotdebouncedebouncecopilotdebounce

exports.getDebounceLimit = async function (e, t) {
  let n;
  if ((await e.get(r.Features).debouncePredict()) && t.measurements.contextualFilterScore) {
    const e = t.measurements.contextualFilterScore;
    const r = .3475;
    const i = 7;
    n = 25 + 250 / (1 + Math.pow(e / r, i));
  } else n = await e.get(r.Features).debounceMs();
  return n > 0 ? n : 75;
};

copilotdebounce75ms

contexualFilterScore copilot

35%contextualFilterEnable

5.

copilot

Extract

Extractprompt

getPromptgetPromptprompt

copilotVSCodeAB

async fetchExperiments(e, t) {
      const n = e.get(r.Fetcher);
      let o;
      try {
        o = await n.fetch("https://default.exp-tas.com/vscode/ab", {
          method: "GET",
          headers: t
        });
      } catch (t) {
        return i.ExpConfig.createFallbackConfig(e, `Error fetching ExP config: ${t}`);
      }
      if (!o.ok) return i.ExpConfig.createFallbackConfig(e, `ExP responded with ${o.status}`);
      const s = await o.json(),
        a = s.Configs.find(e => "vscode" === e.Id) ?? {
          Id: "vscode",
          Parameters: {}
        },
        c = Object.entries(a.Parameters).map(([e, t]) => e + (t ? "" : "cf"));
      return new i.ExpConfig(a.Parameters, s.AssignmentContext, c.join(";"));
    }

abcopilot

Copilot

suffixPercent15.
fimSuffixLengthThreshold0
maxPromptCompletionTokens2048
neighboringTabsOptioneager
neighboringSnippetTypesNeighboringSnippets
numberOfSnippets4
snippetPercent0
suffixStartModeCursorTrimStart
tokenizerName cushman002
indentationMinLengthundefined
indentationMaxLengthundefined
cursorContextFixfalse

PromptgetPrompt

getPrompt

languageMarkerTop
pathMarkerTop
localImportContextDeclarations
snippetPositionTopOfText
lineEndingConvertToUnix
suffixMatchThreshold0
suffixMatchCriteria``Levenshtein
cursorSnippetsPickingStrategy``CursorJaccard

prompt

CopilotpromptPromptElementKind

BeforeCursor
AfterCursor
SimilarFile
ImportedFileimport
LanguageMarkder
PathMarker

PromptElement

CopilotElement

class Priorities {
  constructor() {
    this.registeredPriorities = [0, 1];
  }
  register(e) {
    if (e > Priorities.TOP || e < Priorities.BOTTOM) throw new Error("Priority must be between 0 and 1");
    this.registeredPriorities.push(e);
    return e;
  }
  justAbove(...e) {
    const t = Math.max(...e);
    const n = Math.min(...this.registeredPriorities.filter(e => e > t));
    return this.register((n + t) / 2);
  }
  justBelow(...e) {
    const t = Math.min(...e);
    const n = Math.max(...this.registeredPriorities.filter(e => e < t));
    return this.register((n + t) / 2);
  }
  between(e, t) {
    if (this.registeredPriorities.some(n => n > e && n < t) || !this.registeredPriorities.includes(e) || !this.registeredPriorities.includes(t)) throw new Error("Priorities must be adjacent in the list of priorities");
    return this.register((e + t) / 2);
  }
}

justAbovejustBelow

Copilot

const beforeCursorPriority = priorities.justBelow(p.Priorities.TOP);
  const languageMarkerPriority =
    promptOpts.languageMarker === h.Always
      ? priorities.justBelow(p.Priorities.TOP)
      : priorities.justBelow(beforeCursorPriority);
  const pathMarkerPriority =
    promptOpts.pathMarker === f.Always ? priorities.justBelow(p.Priorities.TOP) : priorities.justBelow(beforeCursorPriority);
  const importedFilePriority = priorities.justBelow(beforeCursorPriority);
  const lowSnippetPriority = priorities.justBelow(importedFilePriority);
  const highSnippetPriority = priorities.justAbove(beforeCursorPriority);

beforeCursorPriority0.5
languageMarkerPriority0.25
pathMarkderPriority0.375
importedFilePriority0.4375
lowSnippetPriority0.40625
highSnippetPriority0.75

highSnippetPriority > beforeCursorPriority > importedFilePriority > lowSnippetPriority > pathMarkderPriority > languageMarkerPriority

PromptElement

languageMarkerpathMarker

languageMarkerpathMarkerpromptWishListlanguageMarkerpathMarker

  if (promptOpts.languageMarker !== h.NoMarker) {
      const e = newLineEnded(r.getLanguageMarker(resourceInfo));
      languageMarkerId = promptWishlist.append(e, p.PromptElementKind.LanguageMarker, languageMarkerPriority);
    }
    if (promptOpts.pathMarker !== f.NoMarker) {
      const e = newLineEnded(r.getPathMarker(resourceInfo));
      if (e.length > 0) {
        pathMarkerId = promptWishlist.append(e, p.PromptElementKind.PathMarker, pathMarkerPriority);
      }
    }

getLanguageMarker

```jsx
exports.getLanguageMarker = function (e) {
  const {
    languageId: t
  } = e;
  return -1 !== n.indexOf(t) || hasLanguageMarker(e) ? "" : t in r ? r[t] : comment(`Language: ${t}`, t);
};
```

languageIdignoreListcopilot

```jsx
const n = ["php", "plaintext"];
```

MapHTMLPythonRubyShellYAML

```jsx
const r = {
  html: "<!DOCTYPE html>",
  python: "#!/usr/bin/env python3",
  ruby: "#!/usr/bin/env ruby",
  shellscript: "#!/bin/sh",
  yaml: "# YAML data"
};
```



```jsx
// Language: ${languageId}
```

getPathMarker

```jsx
exports.getPathMarker = function (e) {
  return e.relativePath ? comment(`Path: ${e.relativePath}`, e.languageId) : "";
};
```

localImportContext

localImportContext

if (promptOpts.localImportContext !== y.NoContext)
    for (const e of await i.extractLocalImportContext(resourceInfo, promptOpts.fs))
      promptWishlist.append(newLineEnded(e), p.PromptElementKind.ImportedFile, importedFilePriority);

extractLocalImportContext

const reg = /^\s*import\s*(type|)\s*\{[^}]*\}\s*from\s*['"]\./gm;
  exports.extractLocalImportContext = async function (resourceInfo, fs) {
    let {
      source: source,
      uri: uri,
      languageId: languageId
    } = resourceInfo;
    return fs && "typescript" === languageId ? async function (source, uri, fs) {
      let language = "typescript";
      let result = [];
      const importEndIndex = function (source) {
        let match;
        let lastIndex = -1;
        reg.lastIndex = -1;
        do {
            match = reg.exec(source);
          if (match) {
            lastIndex = reg.lastIndex + match.length;
          }
        } while (match);
        if (-1 === lastIndex) return -1;
        const nextNewLine = source.indexOf("\n", lastIndex);
        return -1 !== nextNewLine ? nextNewLine : source.length;
      }(source);
      if (-1 === importEndIndex) return result;
      source = source.substring(0, importEndIndex);
      let ast = await i.parseTreeSitter(language, source);
      try {
        for (let node of function (node) {
          let t = [];
          for (let childNode of node.namedChildren) if ("import_statement" === childNode.type) {
            t.push(childNode);
          }
          return t;
        }(ast.rootNode)) {
          let filePath = getTSFilePath(uri, node);
          if (!filePath) continue;
          let namedImports = parseNamedImport(node);
          if (0 === namedImports.length) continue;
          let exports = await getExports(filePath, language, fs);
          for (let e of namedImports) if (exports.has(e.name)) {
            result.push(...exports.get(e.name));
          }
        }
      } finally {
        ast.delete();
      }
      return result;
    }(source, uri, fs) : [];
  }

TypescriptCopilottsimport

importcopilotwasmtree-sitter

copilotimportnamed importexportexportextract

snippets

snippetsCopilotsnippets

const snippets = [
    ...retrievalSnippets,
    ...(promptOpts.neighboringTabs === a.NeighboringTabsOption.None || 0 === neighborDocs.length
      ? []
      : await a.getNeighborSnippets(
          resourceInfo,
          neighborDocs,
          promptOpts.neighboringSnippetTypes,
          promptOpts.neighboringTabs,
          promptOpts.cursorContextFix,
          promptOpts.indentationMinLength,
          promptOpts.indentationMaxLength,
          promptOpts.snippetSelection,
          promptOpts.snippetSelectionK,
          lineCursorHistory,
          promptOpts.cursorSnippetsPickingStrategy
        )),
  ];

retrievalSnippets``neighboringTabs``eager``getNeighborSnippets

neighborDocsextract

let neighborDocs = [];
      let neighborSource = new Map();
      try {
        const t = await u.NeighborSource.getNeighborFiles(ctx, uri, repoUserData);
        neighborDocs = t.docs;
        neighborSource = t.neighborSource;
      } catch (t) {
        telemetry.telemetryException(ctx, t, "prompt.getPromptForSource.exception");
      }

fileTypeOpenTabsVSCodetab()

exports.OpenTabFiles = class {
  constructor(e, t) {
    this.docManager = e;
    this.neighboringLanguageType = t;
  }
  async truncateDocs(e, t, n, r) {
    const o = [];
    let s = 0;
    for (const a of e) if (!(s + a.getText().length > i.NeighborSource.MAX_NEIGHBOR_AGGREGATE_LENGTH) && ("file" == a.uri.scheme && a.fileName !== t && i.considerNeighborFile(n, a.languageId, this.neighboringLanguageType) && (o.push({
      uri: a.uri.toString(),
      relativePath: await this.docManager.getRelativePath(a),
      languageId: a.languageId,
      source: a.getText()
    }), s += a.getText().length), o.length >= r)) break;
    return o;
  }
  async getNeighborFiles(e, t, n, o) {
    let s = [];
    const a = new Map();
    s = await this.truncateDocs(utils2.sortByAccessTimes(this.docManager.textDocuments), e.fsPath, n, o);
    a.set(i.NeighboringFileType.OpenTabs, s.map(e => e.uri));
    return {
      docs: s,
      neighborSource: a
    };
  }
};

getNeighborSnippets

exports.getNeighborSnippets = async function (
  resourceInfo,
  neighborDocs,
  neighboringSnippetTypes,
  neighboringTabs,
  cursorContextFix,
  indentationMinLength,
  indentationMaxLength,
  snippetSelection,
  snippetSelectionK,
  lineCursorHistory,
  cursorSnippetsPickingStrategy
) {
  const options = {
    ...exports.neighborOptionToSelection[neighboringTabs],
  };
  const y = (function (
    resourceInfo,
    neighboringSnippetTypes,
    options,
    cursorContextFix,
    indentationMinLength,
    indentationMaxLength,
    lineCursorHistory,
    cursorSnippetsPickingStrategy = i.CursorSnippetsPickingStrategy
      .CursorJaccard
  ) {
    let d;
    if (neighboringSnippetTypes === s.NeighboringSnippets) {
      d =
        void 0 !== indentationMinLength && void 0 !== indentationMaxLength
          ? o.IndentationBasedJaccardMatcher.FACTORY(
              indentationMinLength,
              indentationMaxLength,
              cursorContextFix
            )
          : o.FixedWindowSizeJaccardMatcher.FACTORY(
              options.snippetLength,
              cursorContextFix
            );
    } else {
      if (neighboringSnippetTypes === s.NeighboringFunctions) {
        d = o.FunctionJaccardMatcher.FACTORY(
          options.snippetLength,
          cursorContextFix,
          indentationMinLength,
          indentationMaxLength
        );
      } else {
        r.ok(
          void 0 !== lineCursorHistory,
          "lineCursorHistory should not be undefined"
        );
        d = i.CursorHistoryMatcher.FACTORY(
          options.snippetLength,
          lineCursorHistory,
          cursorSnippetsPickingStrategy,
          cursorContextFix
        );
      }
    }
    return d.to(resourceInfo);
  })(
    resourceInfo,
    neighboringSnippetTypes,
    options,
    cursorContextFix,
    indentationMinLength,
    indentationMaxLength,
    lineCursorHistory,
    cursorSnippetsPickingStrategy
  );
  return 0 === options.numberOfSnippets
    ? []
    : (
        await neighborDocs
          .filter((e) => e.source.length < 1e4 && e.source.length > 0)
          .slice(0, 20)
          .reduce(
            async (e, t) =>
              (
                await e
              ).concat(
                (
                  await y.findMatches(t, snippetSelection, snippetSelectionK)
                ).map((e) => ({
                  relativePath: t.relativePath,
                  ...e,
                }))
              ),
            Promise.resolve([])
          )
      )
        .filter((e) => e.score && e.snippet && e.score > options.threshold)
        .sort((e, t) => e.score - t.score)
        .slice(-options.numberOfSnippets);
};

neighboringSnippetTypesNeighboringSnippetsFixedWindowSizeJaccardMatcher
neighborDocsfindMatchers
scorethreshold0
score4numberOfSnippets4

FixedWindowSizeJaccardMatcher

class FixedWindowSizeJaccardMatcher extends i.WindowedMatcher {
    constructor(resourceInfo, snippetLength, cursorContextFix) {
      super(resourceInfo, cursorContextFix);
      this.windowLength = snippetLength;
      this.cursorContextFix = cursorContextFix;
    }
    id() {
      return "fixed:" + this.windowLength;
    }
    getWindowsDelineations(e) {
      return o.getBasicWindowDelineations(this.windowLength, e);
    }
    trimDocument(e) {
      return e.source.slice(0, e.offset).split("\n").slice(-this.windowLength).join("\n");
    }
    _getCursorContextInfo(e) {
      return r.getCursorContext(e, {
        maxLineCount: this.windowLength,
        cursorContextFix: this.cursorContextFix
      });
    }
    similarityScore(e, t) {
      return computeScore(e, t);
    }
  }

snippetLength eager60snippet60

WindowedMatcherfindMatchesWindowedMatcher

async findMatches(e, t = i.SnippetSelectionOption.BestMatch, n) {
      if (t == i.SnippetSelectionOption.BestMatch) {
        const t = await this.findBestMatch(e);
        return t ? [t] : [];
      }
      return t == i.SnippetSelectionOption.TopK && (await this.findTopKMatches(e, n)) || [];
    }

undefinedBestMatch

async findBestMatch(e) {
      if (0 === e.source.length || 0 === this.referenceTokens.size) return;
      const t = e.source.split("\n");
      const n = this.retrieveAllSnippets(e, s.Descending);
      return 0 !== n.length && 0 !== n[0].score ? {
        snippet: t.slice(n[0].startLine, n[0].endLine).join("\n"),
        semantics: o.SnippetSemantics.Snippet,
        provider: o.SnippetProvider.NeighboringTabs,
        ...n[0]
      } : void 0;
    }

BestMatchretrieveAllSnippets 0snippet

retrieveAllSnippets(e, t = s.Descending) {
      const n = [];
      if (0 === e.source.length || 0 === this.referenceTokens.size) return n;
      const sourceArr = e.source.split("\n");
      const key = this.id() + ":" + e.source;
      const result = c.get(key) ?? [];
      const noCache = 0 == result.length;
      const tokens = noCache ? sourceArr.map(this.tokenizer.tokenize, this.tokenizer) : [];
      for (const [index, [startLine, endLine]] of this.getWindowsDelineations(sourceArr).entries()) {
        if (noCache) {
          const e = new Set();
          tokens.slice(startLine, endLine).forEach(t => t.forEach(e.add, e));
          result.push(e);
        }
        const r = result[index];
        const s = this.similarityScore(r, this.referenceTokens);
        n.push({
          score: s,
          startLine: startLine,
          endLine: endLine
        });
      }
      if (noCache) {
        c.put(key, result);
      }
      return this.sortScoredSnippets(n, t);
    }

tokenizetoken
getWindowsDelineations1
tokenreferenceDoctokenJaccard

tokenizetoken

const p = new Set(["we", "our", "you", "it", "its", "they", "them", "their", "this", "that", "these", "those", "is", "are", "was", "were", "be", "been", "being", "have", "has", "had", "having", "do", "does", "did", "doing", "can", "don", "t", "s", "will", "would", "should", "what", "which", "who", "when", "where", "why", "how", "a", "an", "the", "and", "or", "not", "no", "but", "because", "as", "until", "again", "further", "then", "once", "here", "there", "all", "any", "both", "each", "few", "more", "most", "other", "some", "such", "above", "below", "to", "during", "before", "after", "of", "at", "by", "about", "between", "into", "through", "from", "up", "down", "in", "out", "on", "off", "over", "under", "only", "own", "same", "so", "than", "too", "very", "just", "now"]);
const d = new Set(["if", "then", "else", "for", "while", "with", "def", "function", "return", "TODO", "import", "try", "catch", "raise", "finally", "repeat", "switch", "case", "match", "assert", "continue", "break", "const", "class", "enum", "struct", "static", "new", "super", "this", "var", ...p]);

tokenize(e) {
  return new Set(splitIntoWords(e).filter(e => !this.stopsForLanguage.has(e)));
}

function splitIntoWords(e) {
  return e.split(/[^a-zA-Z0-9]/).filter(e => e.length > 0);
}

tokensiffor

getWindowsDelineations

exports.getBasicWindowDelineations = function (e, t) {
  const n = [];
  const r = t.length;
  if (0 == r) return [];
  if (r < e) return [[0, r]];
  for (let t = 0; t < r - e + 1; t++) n.push([t, t + e]);
  return n;
};

getWindowsDelineations windowSize1windowSize

windowSizesnippet

get referenceTokens() {
  if (void 0 === this._referenceTokens) {
    this._referenceTokens = this.tokenizer.tokenize(this._getCursorContextInfo(this.referenceDoc).context);
  }
  return this._referenceTokens;
}

exports.getCursorContext = function e(doc, opts = {}) {
    const opts = function (e) {
      return {
        ...i,
        ...e
      };
    }(opts);
    const s = r.getTokenizer(opts.tokenizerName);
    
    if (void 0 === opts.maxTokenLength && void 0 !== opts.maxLineCount) {
      const e = doc.source.slice(0, doc.offset).split("\n").slice(-opts.maxLineCount);
      const n = e.join("\n");
      return {
        context: n,
        lineCount: e.length,
        tokenLength: s.tokenLength(n),
        tokenizerName: opts.tokenizerName
      };
    }
		// ...
  };

tokentoken

Jaccard

Copilot**Jaccard**

function computeScore(e, t) {
    const n = new Set();
    e.forEach(e => {
      if (t.has(e)) {
        n.add(e);
      }
    });
    return n.size / (e.size + t.size - n.size);
  }

Jaccard

Copilot

copilotprocessSnippetsForWishlistsnippetwishList

function $() {
    const maxSnippetLength = Math.round((promptOpts.snippetPercent / 100) * promptOpts.maxPromptLength);
    c.processSnippetsForWishlist(
      snippets,
      resourceInfo.languageId,
      tokenizer,
      promptOpts.snippetProviderOptions,
      {
        priorities: priorities,
        low: lowSnippetPriority,
        high: highSnippetPriority,
      },
      promptOpts.numberOfSnippets,
      maxSnippetLength
    ).forEach((e) => {
      let t = p.PromptElementKind.SimilarFile;
      if (e.provider === c.SnippetProvider.Retrieval) {
        t = p.PromptElementKind.RetrievalSnippet;
      } else {
        if (e.provider == c.SnippetProvider.SymbolDef) {
          t = p.PromptElementKind.SymbolDefinition;
        }
      }
      promptWishlist.append(e.announcedSnippet, t, e.priority, e.tokens, e.normalizedScore);
    });
  }

snippetPercent0maxSnippetLength0.

processSnippetsForWishList

exports.processSnippetsForWishlist = function (snippets, languageId, tokenizer, snippetProviderOptions, priorities, numberOfSnippets, maxSnippetLength) {
    const {
      reserved: reserved,
      candidates: candidates
    } = selectSnippets(snippets, numberOfSnippets, snippetProviderOptions);
    let d = 0;
    let h = [];
    let highPriorities = priorities.high;
    let lowPriorities = priorities.low;
    function g(snippet, r) {
      const o = announceSnippet(snippet, languageId);
      const c = tokenizer.tokenLength(o);
      let l;
      if (r + c <= maxSnippetLength) {
        l = highPriorities;
        highPriorities = priorities.priorities.justBelow(l);
      } else {
        l = lowPriorities;
        lowPriorities = priorities.priorities.justBelow(l);
      }
      h.push({
        announcedSnippet: o,
        provider: snippet.provider,
        providerScore: snippet.providerScore,
        normalizedScore: snippet.normalizedScore,
        priority: l,
        tokens: c,
        relativePath: snippet.relativePath
      });
      return r + c;
    }
    for (const snippet of [...reserved, ...candidates]) {
      if (h.length >= numberOfSnippets) break;
      d = g(snippete, d);
    }
    l(h);
    h.reverse();
    return h;
  };

maxSnippetLengthPrioritylowPriority

scoreannouncedSnippetannounceSnippetPrompt

function announceSnippet(e, t) {
    const n = s[e.semantics];
    let i = (e.relativePath ? `Compare this ${n} from ${e.relativePath}:` : `Compare this ${n}:`) + "\n" + e.snippet;
    if (i.endsWith("\n")) {
      i += "\n";
    }
    return r.commentBlockAsSingles(i, t);
  }

snippetCompare this

beforeCursor

beforeCursor

promptWishlist.appendLineForLine(source.substring(0, offset), p.PromptElementKind.BeforeCursor, beforeCursorPriority).forEach((e) =>
      V.push(e)
    );

appendLineForLineappendappendLineForLine

appendLineForLine(text, kind, priority) {
    const lineArr = (lines = this.convertLineEndings(text)).split("\n");
    for (let i = 0; i < lineArr.length - 1; i++) lineArr[i] += "\n";
    const lines = [];
    lineArr.forEach((line) => {
      if ("\n" === line && lines.length > 0 && !lines[lines.length - 1].endsWith("\n\n")) {
        lines[lines.length - 1] += "\n";
      } else {
        lines.push(line);
      }
    });
    const result = [];
    lines.forEach((text, index) => {
      if ("" !== text) {
        result.push(this.append(text, kind, priority));
        if (index > 0) {
          this.content[this.content.length - 2].requires = [
            this.content[this.content.length - 1],
          ];
        }
      }
    });
    return result;
  }

appendtoken

wishListfullfill

if (h.Top === promptOpts.languageMarker && V.length > 0 && void 0 !== languageMarkerId) {
    promptWishlist.require(languageMarkerId, V[0]);
  }
  if (f.Top === promptOpts.pathMarker && V.length > 0 && void 0 !== pathMarkerId) {
    if (languageMarkerId) {
      promptWishlist.require(pathMarkerId, languageMarkerId);
    } else {
      promptWishlist.require(pathMarkerId, V[0]);
    }
  }
  if (void 0 !== languageMarkerId && void 0 !== pathMarkerId) {
    promptWishlist.exclude(pathMarkerId, languageMarkerId);
  }

pathMarkerlanguageMarkerpathMarkerlanguageMarkerexcludelanguageMarker

suffixPercent0fullfill

if (0 === promptOpts.suffixPercent || q.length <= promptOpts.fimSuffixLengthThreshold)
    return promptWishlist.fulfill(promptOpts.maxPromptLength);

suffixPercent150suffix

fullfillsuffix:

fulfill(maxPromptLength) {
    const promptChoices = new PromptChoices();
    const promptBackground = new PromptBackground();
    const elements = this.content.map((e, t) => ({
      element: e,
      index: t,
    }));
    elements.sort((e, t) =>
      e.element.priority === t.element.priority
        ? t.index - e.index
        : t.element.priority - e.element.priority
    );
    const requires = new Set();
    const excludes = new Set();
    let lastElement;
    const results = [];
    let promptLength = maxPromptLength;
    elements.forEach((e) => {
      const element = e.element;
      const index = e.index;
      if (
        promptLength >= 0 &&
        (promptLength > 0 || void 0 === lastElement) &&
        element.requires.every((e) => requires.has(e.id)) &&
        !excludes.has(r.id)
      ) {
        let tokens = element.tokens;
        const nextElement = (function (e, t) {
          let n;
          let r = 1 / 0;
          for (const i of e)
            if (i.index > t && i.index < r) {
              n = i;
              r = i.index;
            }
          return n;
        })(results, index)?.element;
        if (element.text.endsWith("\n\n") && nextElement && !nextElement.text.match(/^\s/)) {
          tokens++;
        }
        if (promptLength >= tokens) {
            promptLength -= tokens;
            requires.add(r.id);
          element.excludes.forEach((e) => excludes.add(e.id));
          promptChoices.markUsed(element);
          promptBackground.markUsed(element);
          results.push(e);
        } else {
          if (void 0 === lastElement) {
            lastElement = e;
          } else {
            promptChoices.markUnused(e.element);
            promptBackground.markUnused(e.element);
          }
        }
      } else {
        promptChoices.markUnused(element);
        promptBackground.markUnused(element);
      }
    });
    results.sort((e, t) => e.index - t.index);
    let prefix = results.reduce((e, t) => e + t.element.text, "");
    let prefixLength = this.tokenizer.tokenLength(prefix);
    for (; prefixLength > maxPromptLength; ) {
      u.sort((e, t) =>
        t.element.priority === e.element.priority
          ? t.index - e.index
          : t.element.priority - e.element.priority
      );
      const e = u.pop();
      if (e) {
        promptChoices.undoMarkUsed(e.element);
        promptChoices.markUnused(e.element);
        promptBackground.undoMarkUsed(e.element);
        promptBackground.markUnused(e.element);
        if (void 0 !== lastElement) {
          promptChoices.markUnused(lastElement.element);
          promptBackground.markUnused(lastElement.element);
        }
        lastElement = void 0;
      }
      u.sort((e, t) => e.index - t.index);
      prefix = u.reduce((e, t) => e + t.element.text, "");
      prefixLength = this.tokenizer.tokenLength(prefix);
    }
    const f = [...u];
    if (void 0 !== lastElement) {
      f.push(lastElement);
      f.sort((e, t) => e.index - t.index);
      const prefix = f.reduce((e, t) => e + t.element.text, "");
      const prefixLength = this.tokenizer.tokenLength(prefix);
      if (prefixLength <= maxPromptLength) {
        promptChoices.markUsed(l.element);
        promptBackground.markUsed(l.element);
        const promptElementRanges = new PromptElementRanges(f);
        return {
          prefix: prefix,
          suffix: "",
          prefixLength: prefixLength,
          suffixLength: 0,
          promptChoices: promptChoices,
          promptBackground: promptBackground,
          promptElementRanges: promptElementRanges,
        };
      }
      promptChoices.markUnused(l.element);
      promptBackground.markUnused(l.element);
    }
    const m = new PromptElementRanges(u);
    return {
      prefix: prefix,
      suffix: "",
      prefixLength: prefixLength,
      suffixLength: 0,
      promptChoices: promptChoices,
      promptBackground: promptBackground,
      promptElementRanges: m,
    };
  }