refactor: rewrite installation using jobs and async

Replace sync variants with callback support

9 files changed, 930 insertions, 650 deletions

diff --git a/lua/nvim-treesitter/async.lua b/lua/nvim-treesitter/async.lua
new file mode 100644
index 000000000..57a670ef6
--- /dev/null
+++ b/lua/nvim-treesitter/async.lua
@@ -0,0 +1,110 @@
+local co = coroutine
+
+local M = {}
+
+---Executes a future with a callback when it is done
+--- @param func function
+--- @param callback function
+--- @param ... unknown
+local function execute(func, callback, ...)
+  local thread = co.create(func)
+
+  local function step(...)
+    local ret = { co.resume(thread, ...) }
+    --- @type boolean, any
+    local stat, nargs_or_err = unpack(ret)
+
+    if not stat then
+      error(
+        string.format(
+          'The coroutine failed with this message: %s\n%s',
+          nargs_or_err,
+          debug.traceback(thread)
+        )
+      )
+    end
+
+    if co.status(thread) == 'dead' then
+      if callback then
+        callback(unpack(ret, 3, table.maxn(ret)))
+      end
+      return
+    end
+
+    --- @type function, any[]
+    local fn, args = ret[3], { unpack(ret, 4, table.maxn(ret)) }
+    args[nargs_or_err] = step
+    fn(unpack(args, 1, nargs_or_err))
+  end
+
+  step(...)
+end
+
+--- Creates an async function with a callback style function.
+--- @generic F: function
+--- @param func F
+--- @param argc integer
+--- @return F
+function M.wrap(func, argc)
+  --- @param ... unknown
+  --- @return unknown
+  return function(...)
+    return co.yield(argc, func, ...)
+  end
+end
+
+---Use this to create a function which executes in an async context but
+---called from a non-async context. Inherently this cannot return anything
+---since it is non-blocking
+--- @generic F: function
+--- @param func async F
+--- @param nargs? integer
+--- @return F
+function M.sync(func, nargs)
+  nargs = nargs or 0
+  return function(...)
+    local callback = select(nargs + 1, ...)
+    execute(func, callback, unpack({ ... }, 1, nargs))
+  end
+end
+
+--- @param n integer max number of concurrent jobs
+--- @param interrupt_check? function
+--- @param thunks function[]
+--- @return any
+function M.join(n, interrupt_check, thunks)
+  return co.yield(1, function(finish)
+    if #thunks == 0 then
+      return finish()
+    end
+
+    local remaining = { select(n + 1, unpack(thunks)) }
+    local to_go = #thunks
+
+    local ret = {} --- @type any[]
+
+    local function cb(...)
+      ret[#ret + 1] = { ... }
+      to_go = to_go - 1
+      if to_go == 0 then
+        finish(ret)
+      elseif not interrupt_check or not interrupt_check() then
+        if #remaining > 0 then
+          local next_task = table.remove(remaining)
+          next_task(cb)
+        end
+      end
+    end
+
+    for i = 1, math.min(n, #thunks) do
+      thunks[i](cb)
+    end
+  end, 1)
+end
+
+---An async function that when called will yield to the Neovim scheduler to be
+---able to call the API.
+--- @type fun()
+M.main = M.wrap(vim.schedule, 1)
+
+return M
diff --git a/lua/nvim-treesitter/config.lua b/lua/nvim-treesitter/config.lua
index 01880a8bc..b8a9f6e91 100644
--- a/lua/nvim-treesitter/config.lua
+++ b/lua/nvim-treesitter/config.lua
@@ -1,7 +1,6 @@
 local M = {}
 
 ---@class TSConfig
----@field sync_install boolean
 ---@field auto_install boolean
 ---@field ensure_install string[]
 ---@field ignore_install string[]
@@ -9,7 +8,6 @@ local M = {}
 
 ---@type TSConfig
 local config = {
-  sync_install = false,
   auto_install = false,
   ensure_install = {},
   ignore_install = {},
@@ -38,7 +36,11 @@ function M.setup(user_data)
           and not vim.list_contains(M.installed_parsers(), lang)
           and not vim.list_contains(config.ignore_install, lang)
         then
-          require('nvim-treesitter.install').install(lang)
+          require('nvim-treesitter.install').install(lang, nil, function()
+            -- Need to pcall since 'FileType' can be triggered multiple times
+            -- per buffer
+            pcall(vim.treesitter.start, args.buf, lang)
+          end)
         end
       end,
     })
@@ -48,9 +50,7 @@ function M.setup(user_data)
     local to_install = M.norm_languages(config.ensure_install, { ignored = true, installed = true })
 
     if #to_install > 0 then
-      require('nvim-treesitter.install').install(to_install, {
-        with_sync = config.sync_install,
-      })
+      require('nvim-treesitter.install').install(to_install)
     end
   end
 end
@@ -63,9 +63,10 @@ function M.get_install_dir(dir_name)
   local dir = vim.fs.joinpath(config.install_dir, dir_name)
 
   if not vim.loop.fs_stat(dir) then
-    local ok, error = pcall(vim.fn.mkdir, dir, 'p', '0755')
+    local ok, err = pcall(vim.fn.mkdir, dir, 'p', '0755')
     if not ok then
-      vim.notify(error, vim.log.levels.ERROR)
+      local log = require('nvim-treesitter.log')
+      log.error(err --[[@as string]])
     end
   end
   return dir
@@ -105,35 +106,37 @@ function M.norm_languages(languages, skip)
     end
     languages = parsers.get_available()
   end
+  --TODO(clason): skip and warn on unavailable parser
 
   for i, tier in ipairs(parsers.tiers) do
     if vim.list_contains(languages, tier) then
       languages = vim.iter.filter(function(l)
         return l ~= tier
-      end, languages)
+      end, languages) --[[@as string[] ]]
       vim.list_extend(languages, parsers.get_available(i))
     end
   end
 
+  --TODO(clason): support skipping tiers
   if skip and skip.ignored then
     local ignored = config.ignore_install
     languages = vim.iter.filter(function(v)
       return not vim.list_contains(ignored, v)
-    end, languages)
+    end, languages) --[[@as string[] ]]
   end
 
   if skip and skip.installed then
     local installed = M.installed_parsers()
     languages = vim.iter.filter(function(v)
       return not vim.list_contains(installed, v)
-    end, languages)
+    end, languages) --[[@as string[] ]]
   end
 
   if skip and skip.missing then
     local installed = M.installed_parsers()
     languages = vim.iter.filter(function(v)
       return vim.list_contains(installed, v)
-    end, languages)
+    end, languages) --[[@as string[] ]]
   end
 
   return languages
diff --git a/lua/nvim-treesitter/health.lua b/lua/nvim-treesitter/health.lua
index e8c945f6c..1e17bb00b 100644
--- a/lua/nvim-treesitter/health.lua
+++ b/lua/nvim-treesitter/health.lua
@@ -1,6 +1,6 @@
-local shell = require('nvim-treesitter.shell_cmds')
 local install = require('nvim-treesitter.install')
 local config = require('nvim-treesitter.config')
+local util = require('nvim-treesitter.util')
 local tsq = vim.treesitter.query
 
 local M = {}
@@ -62,7 +62,7 @@ local function install_health()
     vim.health.ok('`git` executable found.')
   end
 
-  local cc = shell.select_executable(install.compilers)
+  local cc = install.select_executable(install.compilers)
   if not cc then
     vim.health.error('`cc` executable not found.', {
       'Check that any of '
@@ -118,6 +118,7 @@ local function query_status(lang, query_group)
 end
 
 function M.check()
+  --- @type {[1]: string, [2]: string, [3]: string}[]
   local error_collection = {}
   -- Installation dependency checks
   install_health()
@@ -144,22 +145,19 @@ function M.check()
   if #error_collection > 0 then
     vim.health.start('The following errors have been detected:')
     for _, p in ipairs(error_collection) do
-      local lang, type, err = unpack(p)
+      local lang, type, err = p[1], p[2], p[3]
       local lines = {}
       table.insert(lines, lang .. '(' .. type .. '): ' .. err)
       local files = tsq.get_files(lang, type)
       if #files > 0 then
         table.insert(lines, lang .. '(' .. type .. ') is concatenated from the following files:')
         for _, file in ipairs(files) do
-          local fd = io.open(file, 'r')
-          if fd then
-            local ok, file_err = pcall(tsq.parse, lang, fd:read('*a'))
-            if ok then
-              table.insert(lines, '|  [OK]:"' .. file .. '"')
-            else
-              table.insert(lines, '| [ERR]:"' .. file .. '", failed to load: ' .. file_err)
-            end
-            fd:close()
+          local query = util.read_file(file)
+          local ok, file_err = pcall(tsq.parse, lang, query)
+          if ok then
+            table.insert(lines, '|  [OK]:"' .. file .. '"')
+          else
+            table.insert(lines, '| [ERR]:"' .. file .. '", failed to load: ' .. file_err)
           end
         end
       end
diff --git a/lua/nvim-treesitter/install.lua b/lua/nvim-treesitter/install.lua
index 0db61137c..ab9543673 100644
--- a/lua/nvim-treesitter/install.lua
+++ b/lua/nvim-treesitter/install.lua
@@ -2,194 +2,56 @@ local api = vim.api
 local fs = vim.fs
 local uv = vim.loop
 
-local parsers = require('nvim-treesitter.parsers')
+local a = require('nvim-treesitter.async')
 local config = require('nvim-treesitter.config')
-local shell = require('nvim-treesitter.shell_cmds')
-
-local M = {}
-
----@class LockfileInfo
----@field revision string
-
----@type table<string, LockfileInfo>
-local lockfile = {}
-
-M.compilers = { uv.os_getenv('CC'), 'cc', 'gcc', 'clang', 'cl', 'zig' }
-M.prefer_git = uv.os_uname().sysname == 'Windows_NT'
-M.command_extra_args = {}
-M.ts_generate_args = nil
-
-local started_commands = 0
-local finished_commands = 0
-local failed_commands = 0
-local stdout_output = {}
-local stderr_output = {}
-
----
---- JOB API functions
----
+local job = require('nvim-treesitter.job')
+local log = require('nvim-treesitter.log')
+local parsers = require('nvim-treesitter.parsers')
+local util = require('nvim-treesitter.util')
 
-local function reset_progress_counter()
-  if started_commands ~= finished_commands then
-    return
-  end
-  started_commands = 0
-  finished_commands = 0
-  failed_commands = 0
-  stdout_output = {}
-  stderr_output = {}
-end
+--- @type fun(path: string, new_path: string, flags?: table): string?
+local uv_copyfile = a.wrap(uv.fs_copyfile, 4)
 
-local function get_job_status()
-  return '[nvim-treesitter] ['
-    .. finished_commands
-    .. '/'
-    .. started_commands
-    .. (failed_commands > 0 and ', failed: ' .. failed_commands or '')
-    .. ']'
-end
+--- @type fun(path: string, mode: integer): string?
+local uv_mkdir = a.wrap(uv.fs_mkdir, 3)
 
----@param cmd Command
----@return string command
-local function get_command(cmd)
-  local options = ''
-  if cmd.opts and cmd.opts.args then
-    if M.command_extra_args[cmd.cmd] then
-      vim.list_extend(cmd.opts.args, M.command_extra_args[cmd.cmd])
-    end
-    for _, opt in ipairs(cmd.opts.args) do
-      options = string.format('%s %s', options, opt)
-    end
-  end
+--- @type fun(path: string, new_path: string): string?
+local uv_rename = a.wrap(uv.fs_rename, 3)
 
-  local command = string.format('%s %s', cmd.cmd, options)
-  if cmd.opts and cmd.opts.cwd then
-    command = shell.make_directory_change_for_command(cmd.opts.cwd, command)
-  end
-  return command
-end
+--- @type fun(path: string, new_path: string, flags?: table): string?
+local uv_symlink = a.wrap(uv.fs_symlink, 4)
 
----@param cmd_list Command[]
----@return boolean
-local function iter_cmd_sync(cmd_list)
-  for _, cmd in ipairs(cmd_list) do
-    if cmd.info then
-      vim.notify(cmd.info)
-    end
+--- @type fun(path: string): string?
+local uv_unlink = a.wrap(uv.fs_unlink, 2)
 
-    if type(cmd.cmd) == 'function' then
-      cmd.cmd()
-    else
-      local ret = vim.fn.system(get_command(cmd))
-      if vim.v.shell_error ~= 0 then
-        vim.notify(ret)
-        api.nvim_err_writeln(
-          (cmd.err and cmd.err .. '\n' or '')
-            .. 'Failed to execute the following command:\n'
-            .. vim.inspect(cmd)
-        )
-        return false
-      end
-    end
-  end
-
-  return true
-end
+local M = {}
 
-local function iter_cmd(cmd_list, i, lang, success_message)
-  if i == 1 then
-    started_commands = started_commands + 1
-  end
-  if i == #cmd_list + 1 then
-    finished_commands = finished_commands + 1
-    return vim.notify(get_job_status() .. ' ' .. success_message)
-  end
+---@class LockfileInfo
+---@field revision string
 
-  local attr = cmd_list[i]
-  if attr.info then
-    vim.notify(get_job_status() .. ' ' .. attr.info)
-  end
+---@type table<string, LockfileInfo>
+local lockfile = {}
 
-  if attr.opts and attr.opts.args and M.command_extra_args[attr.cmd] then
-    vim.list_extend(attr.opts.args, M.command_extra_args[attr.cmd])
-  end
+local max_jobs = 50
 
-  if type(attr.cmd) == 'function' then
-    local ok, err = pcall(attr.cmd)
-    if ok then
-      iter_cmd(cmd_list, i + 1, lang, success_message)
-    else
-      failed_commands = failed_commands + 1
-      finished_commands = finished_commands + 1
-      return api.nvim_err_writeln(
-        (attr.err or ('Failed to execute the following command:\n' .. vim.inspect(attr)))
-          .. '\n'
-          .. vim.inspect(err)
-      )
-    end
-  else
-    local handle
-    local stdout = uv.new_pipe(false)
-    local stderr = uv.new_pipe(false)
-    attr.opts.stdio = { nil, stdout, stderr }
-    ---@type userdata
-    handle = uv.spawn(
-      attr.cmd,
-      attr.opts,
-      vim.schedule_wrap(function(code)
-        if code ~= 0 then
-          stdout:read_stop()
-          stderr:read_stop()
-        end
-        stdout:close()
-        stderr:close()
-        handle:close()
-        if code ~= 0 then
-          failed_commands = failed_commands + 1
-          finished_commands = finished_commands + 1
-          if stdout_output[handle] and stdout_output[handle] ~= '' then
-            vim.notify(stdout_output[handle])
-          end
+local iswin = uv.os_uname().sysname == 'Windows_NT'
+local ismac = uv.os_uname().sysname == 'Darwin'
 
-          local err_msg = stderr_output[handle] or ''
-          api.nvim_err_writeln(
-            'nvim-treesitter['
-              .. lang
-              .. ']: '
-              .. (attr.err or ('Failed to execute the following command:\n' .. vim.inspect(attr)))
-              .. '\n'
-              .. err_msg
-          )
-          return
-        end
-        iter_cmd(cmd_list, i + 1, lang, success_message)
-      end)
-    )
-    uv.read_start(stdout, function(_, data)
-      if data then
-        stdout_output[handle] = (stdout_output[handle] or '') .. data
-      end
-    end)
-    uv.read_start(stderr, function(_, data)
-      if data then
-        stderr_output[handle] = (stderr_output[handle] or '') .. data
-      end
-    end)
-  end
-end
+--- @diagnostic disable-next-line:missing-parameter
+M.compilers = { uv.os_getenv('CC'), 'cc', 'gcc', 'clang', 'cl', 'zig' }
 
 ---
 --- PARSER INFO
 ---
 
 ---@param lang string
----@param validate boolean|nil
+---@param validate? boolean
 ---@return InstallInfo
 local function get_parser_install_info(lang, validate)
   local parser_config = parsers.configs[lang]
 
   if not parser_config then
-    error('Parser not available for language "' .. lang .. '"')
+    log.error('Parser not available for language "' .. lang .. '"')
   end
 
   local install_info = parser_config.install_info
@@ -204,14 +66,18 @@ local function get_parser_install_info(lang, validate)
   return install_info
 end
 
+--- @param ... string
+--- @return string
+function M.get_package_path(...)
+  return fs.joinpath(vim.fn.fnamemodify(debug.getinfo(1, 'S').source:sub(2), ':p:h:h:h'), ...)
+end
+
 ---@param lang string
----@return string|nil
-local function get_revision(lang)
+---@return string?
+local function get_target_revision(lang)
   if #lockfile == 0 then
-    local filename = shell.get_package_path('lockfile.json')
-    local file = assert(io.open(filename, 'r'))
-    lockfile = vim.json.decode(file:read('*all'))
-    file:close()
+    local filename = M.get_package_path('lockfile.json')
+    lockfile = vim.json.decode(util.read_file(filename)) --[[@as table<string, LockfileInfo>]]
   end
 
   local install_info = get_parser_install_info(lang)
@@ -225,19 +91,16 @@ local function get_revision(lang)
 end
 
 ---@param lang string
----@return string|nil
+---@return string?
 local function get_installed_revision(lang)
   local lang_file = fs.joinpath(config.get_install_dir('parser-info'), lang .. '.revision')
-  local file = assert(io.open(lang_file, 'r'))
-  local revision = file:read('*a')
-  file:close()
-  return revision
+  return util.read_file(lang_file)
 end
 
 ---@param lang string
 ---@return boolean
 local function needs_update(lang)
-  local revision = get_revision(lang)
+  local revision = get_target_revision(lang)
   return not revision or revision ~= get_installed_revision(lang)
 end
 
@@ -254,7 +117,7 @@ function M.info()
 
   for _, lang in pairs(parser_list) do
     local parser = (lang .. string.rep(' ', max_len - #lang + 1))
-    local output
+    local output --- @type string[]
     if vim.list_contains(installed, lang) then
       output = { parser .. '[✓] installed', 'DiagnosticOk' }
     elseif #api.nvim_get_runtime_file('parser/' .. lang .. '.*', true) > 0 then
@@ -270,13 +133,311 @@ end
 --- PARSER MANAGEMENT FUNCTIONS
 ---
 
+--- @param repo InstallInfo
+--- @param project_name string
+--- @param cache_dir string
+--- @param from_local_path boolean
+--- @return string
+local function get_compile_location(repo, cache_dir, project_name, from_local_path)
+  ---@type string compile_location only needed for typescript installs.
+  if from_local_path then
+    local compile_location = repo.url
+    if repo.location then
+      compile_location = fs.joinpath(compile_location, repo.location)
+    end
+    return compile_location
+  end
+
+  local repo_location = project_name
+  if repo.location then
+    repo_location = fs.joinpath(repo_location, repo.location)
+  end
+  return fs.joinpath(cache_dir, repo_location)
+end
+
+local function cc_err()
+  log.error('No C compiler found! "' .. table.concat(
+    vim.tbl_filter(
+      ---@param c string
+      ---@return boolean
+      function(c)
+        return type(c) == 'string'
+      end,
+      M.compilers
+    ),
+    '", "'
+  ) .. '" are not executable.')
+end
+
+--- @param logger Logger
+--- @param repo InstallInfo
+--- @param compile_location string
+local function do_generate_from_grammar(logger, repo, compile_location)
+  if repo.generate_requires_npm then
+    if vim.fn.executable('npm') ~= 1 then
+      logger:error('NPM requires to be installed from grammar.js')
+    end
+
+    logger:info('Installing NPM dependencies')
+    local r = job.run({ 'npm', 'install' }, { cwd = compile_location })
+    a.main()
+    if r.exit_code > 0 then
+      logger:error('Error during `npm install`')
+    end
+  end
+
+  logger:info('Generating source files from grammar.js...')
+
+  local r = job.run({
+    vim.fn.exepath('tree-sitter'),
+    'generate',
+    '--no-bindings',
+    '--abi',
+    tostring(vim.treesitter.language_version),
+  }, { cwd = compile_location })
+  a.main()
+  if r.exit_code > 0 then
+    logger:error('Error during "tree-sitter generate"')
+  end
+end
+
+---@param logger Logger
+---@param repo InstallInfo
+---@param project_name string
+---@param cache_dir string
+---@param revision string
+---@param project_dir string
+local function do_download_tar(logger, repo, project_name, cache_dir, revision, project_dir)
+  local is_github = repo.url:find('github.com', 1, true)
+  local url = repo.url:gsub('.git$', '')
+
+  local dir_rev = revision
+  if is_github and revision:find('^v%d') then
+    dir_rev = revision:sub(2)
+  end
+
+  local temp_dir = project_dir .. '-tmp'
+
+  util.delete(temp_dir)
+
+  logger:info('Downloading ' .. project_name .. '...')
+  local target = is_github and url .. '/archive/' .. revision .. '.tar.gz'
+    or url .. '/-/archive/' .. revision .. '/' .. project_name .. '-' .. revision .. '.tar.gz'
+
+  local r = job.run({
+    'curl',
+    '--silent',
+    '--show-error',
+    '-L', -- follow redirects
+    target,
+    '--output',
+    project_name .. '.tar.gz',
+  }, {
+    cwd = cache_dir,
+  })
+  a.main()
+  if r.exit_code > 0 then
+    logger:error(
+      'Error during download, please verify your internet connection: ' .. vim.inspect(r.stderr)
+    )
+  end
+
+  logger:debug('Creating temporary directory: ' .. temp_dir)
+  --TODO(clason): use vim.fn.mkdir(temp_dir, 'p') in case stdpath('cache') is not created
+  local err = uv_mkdir(temp_dir, 493)
+  a.main()
+  if err then
+    logger:error('Could not create %s-tmp: %s', project_name, err)
+  end
+
+  logger:info('Extracting ' .. project_name .. '...')
+  r = job.run({
+    'tar',
+    '-xzf',
+    project_name .. '.tar.gz',
+    '-C',
+    project_name .. '-tmp',
+  }, {
+    cwd = cache_dir,
+  })
+
+  a.main()
+  if r.exit_code > 0 then
+    logger:error('Error during tarball extraction: ' .. vim.inspect(r.stderr))
+  end
+
+  err = uv_unlink(project_dir .. '.tar.gz')
+  if err then
+    logger:error('Could not remove tarball: ' .. err)
+  end
+  a.main()
+
+  err = uv_rename(fs.joinpath(temp_dir, url:match('[^/]-$') .. '-' .. dir_rev), project_dir)
+  a.main()
+
+  if err then
+    logger:error('Could not rename temp: ' .. err)
+  end
+
+  util.delete(temp_dir)
+end
+
+---@param logger Logger
+---@param repo InstallInfo
+---@param project_name string
+---@param cache_dir string
+---@param revision string
+---@param project_dir string
+local function do_download_git(logger, repo, project_name, cache_dir, revision, project_dir)
+  logger:info('Downloading ' .. project_name .. '...')
+
+  local r = job.run({
+    'git',
+    'clone',
+    '--filter=blob:none',
+    repo.url,
+    project_name,
+  }, {
+    cwd = cache_dir,
+  })
+
+  a.main()
+
+  if r.exit_code > 0 then
+    logger:error(
+      'Error during download, please verify your internet connection: ' .. vim.inspect(r.stderr)
+    )
+  end
+
+  logger:info('Checking out locked revision')
+  r = job.run({
+    'git',
+    'checkout',
+    revision,
+  }, {
+    cwd = project_dir,
+  })
+
+  a.main()
+
+  if r.exit_code > 0 then
+    logger:error('Error while checking out revision: ' .. vim.inspect(r.stderr))
+  end
+end
+
+---@param executables string[]
+---@return string?
+function M.select_executable(executables)
+  return vim.tbl_filter(
+    ---@param c string
+    ---@return boolean
+    function(c)
+      return c ~= vim.NIL and vim.fn.executable(c) == 1
+    end,
+    executables
+  )[1]
+end
+
+-- Returns the compiler arguments based on the compiler and OS
+---@param repo InstallInfo
+---@param compiler string
+---@return string[]
+local function select_compiler_args(repo, compiler)
+  if compiler:find('cl$') or compiler:find('cl.exe$') then
+    return {
+      '/Fe:',
+      'parser.so',
+      '/Isrc',
+      repo.files,
+      '-Os',
+      '/utf-8',
+      '/LD',
+    }
+  end
+
+  if compiler:find('zig$') or compiler:find('zig.exe$') then
+    return {
+      'c++',
+      '-o',
+      'parser.so',
+      repo.files,
+      '-lc',
+      '-Isrc',
+      '-shared',
+      '-Os',
+    }
+  end
+
+  local args = {
+    '-o',
+    'parser.so',
+    '-I./src',
+    repo.files,
+    '-Os',
+    ismac and '-bundle' or '-shared',
+  }
+
+  if
+    #vim.tbl_filter(
+      --- @param file string
+      --- @return boolean
+      function(file)
+        local ext = vim.fn.fnamemodify(file, ':e')
+        return ext == 'cc' or ext == 'cpp' or ext == 'cxx'
+      end,
+      repo.files
+    ) > 0
+  then
+    table.insert(args, '-lstdc++')
+  end
+
+  if not iswin then
+    table.insert(args, '-fPIC')
+  end
+
+  return args
+end
+
+---@param repo InstallInfo
+---@return boolean
+local function can_download_tar(repo)
+  local can_use_tar = vim.fn.executable('tar') == 1 and vim.fn.executable('curl') == 1
+  local is_github = repo.url:find('github.com', 1, true)
+  local is_gitlab = repo.url:find('gitlab.com', 1, true)
+  return can_use_tar and (is_github or is_gitlab) and not iswin
+end
+
+-- Returns the compile command based on the OS and user options
+---@param repo InstallInfo
+---@param cc string
+---@param compile_location string
+---@return JobResult
+local function do_compile(repo, cc, compile_location)
+  local make = M.select_executable({ 'gmake', 'make' })
+
+  local cmd --- @type string[]
+  if cc:find('cl$') or cc:find('cl.exe$') or not repo.use_makefile or iswin or not make then
+    local args = vim.tbl_flatten(select_compiler_args(repo, cc))
+    cmd = vim.list_extend({ cc }, args)
+  else
+    cmd = {
+      make,
+      '--makefile=' .. M.get_package_path('scripts', 'compile_parsers.makefile'),
+      'CC=' .. cc,
+    }
+  end
+
+  local r = job.run(cmd, { cwd = compile_location })
+  a.main()
+  return r
+end
+
 ---@param lang string
 ---@param cache_dir string
 ---@param install_dir string
 ---@param force boolean
----@param with_sync boolean
 ---@param generate_from_grammar boolean
-local function install_lang(lang, cache_dir, install_dir, force, with_sync, generate_from_grammar)
+local function install_lang(lang, cache_dir, install_dir, force, generate_from_grammar)
   if vim.list_contains(config.installed_parsers(), lang) then
     if not force then
       local yesno =
@@ -288,278 +449,260 @@ local function install_lang(lang, cache_dir, install_dir, force, with_sync, gene
     end
   end
 
+  local cc = M.select_executable(M.compilers)
+  if not cc then
+    cc_err()
+    return
+  end
+
   local repo = get_parser_install_info(lang)
 
   local project_name = 'tree-sitter-' .. lang
+
+  local logger = log.new('install/' .. lang)
+
+  generate_from_grammar = repo.requires_generate_from_grammar or generate_from_grammar
+
+  if generate_from_grammar and vim.fn.executable('tree-sitter') ~= 1 then
+    logger:error('tree-sitter CLI not found: `tree-sitter` is not executable')
+  end
+
+  if generate_from_grammar and vim.fn.executable('node') ~= 1 then
+    logger:error('Node JS not found: `node` is not executable')
+  end
+
+  local revision = repo.revision or get_target_revision(lang)
+
   local maybe_local_path = fs.normalize(repo.url)
   local from_local_path = vim.fn.isdirectory(maybe_local_path) == 1
   if from_local_path then
     repo.url = maybe_local_path
   end
 
-  ---@type string compile_location only needed for typescript installs.
-  local compile_location
-  if from_local_path then
-    compile_location = repo.url
-    if repo.location then
-      compile_location = fs.joinpath(compile_location, repo.location)
-    end
-  else
-    local repo_location = project_name
-    if repo.location then
-      repo_location = fs.joinpath(repo_location, repo.location)
-    end
-    compile_location = fs.joinpath(cache_dir, repo_location)
-  end
-  local parser_lib_name = fs.joinpath(install_dir, lang) .. '.so'
+  if not from_local_path then
+    util.delete(fs.joinpath(cache_dir, project_name))
+    local project_dir = fs.joinpath(cache_dir, project_name)
 
-  generate_from_grammar = repo.requires_generate_from_grammar or generate_from_grammar
+    revision = revision or repo.branch or 'master'
 
-  if generate_from_grammar and vim.fn.executable('tree-sitter') ~= 1 then
-    api.nvim_err_writeln('tree-sitter CLI not found: `tree-sitter` is not executable')
-    if repo.requires_generate_from_grammar then
-      api.nvim_err_writeln(
-        'tree-sitter CLI is needed because the parser for `'
-          .. lang
-          .. '` needs to be generated from grammar'
-      )
-    end
-    return
-  else
-    if not M.ts_generate_args then
-      M.ts_generate_args = { 'generate', '--no-bindings', '--abi', vim.treesitter.language_version }
+    if can_download_tar(repo) then
+      do_download_tar(logger, repo, project_name, cache_dir, revision, project_dir)
+    else
+      do_download_git(logger, repo, project_name, cache_dir, revision, project_dir)
     end
   end
-  if generate_from_grammar and vim.fn.executable('node') ~= 1 then
-    api.nvim_err_writeln('Node JS not found: `node` is not executable')
-    return
-  end
-  local cc = shell.select_executable(M.compilers)
-  if not cc then
-    api.nvim_err_writeln(
-      'No C compiler found! "'
-        .. table.concat(
-          vim.tbl_filter(function(c) ---@param c string
-            return type(c) == 'string'
-          end, M.compilers),
-          '", "'
-        )
-        .. '" are not executable.'
-    )
-    return
+
+  local compile_location = get_compile_location(repo, cache_dir, project_name, from_local_path)
+
+  if generate_from_grammar then
+    do_generate_from_grammar(logger, repo, compile_location)
   end
 
-  local revision = repo.revision
-  if not revision then
-    revision = get_revision(lang)
+  logger:info('Compiling parser')
+  local r = do_compile(repo, cc, compile_location)
+  if r.exit_code > 0 then
+    logger:error('Error during compilation: ' .. vim.inspect(r.stderr))
   end
 
-  ---@class Command
-  ---@field cmd string
-  ---@field info string
-  ---@field err string
-  ---@field opts CmdOpts
+  local parser_lib_name = fs.joinpath(install_dir, lang) .. '.so'
+
+  local err = uv_copyfile(fs.joinpath(compile_location, 'parser.so'), parser_lib_name)
+  a.main()
+  if err then
+    logger:error(err)
+  end
 
-  ---@class CmdOpts
-  ---@field args string[]
-  ---@field cwd string
+  local revfile = fs.joinpath(config.get_install_dir('parser-info') or '', lang .. '.revision')
+  util.write_file(revfile, revision or '')
 
-  ---@type Command[]
-  local command_list = {}
   if not from_local_path then
-    vim.list_extend(command_list, {
-      {
-        cmd = function()
-          vim.fn.delete(fs.joinpath(cache_dir, project_name), 'rf')
-        end,
-      },
-    })
-    vim.list_extend(
-      command_list,
-      shell.select_download_commands(repo, project_name, cache_dir, revision, M.prefer_git)
-    )
-  end
-  if generate_from_grammar then
-    if repo.generate_requires_npm then
-      if vim.fn.executable('npm') ~= 1 then
-        api.nvim_err_writeln('`' .. lang .. '` requires NPM to be installed from grammar.js')
-        return
-      end
-      vim.list_extend(command_list, {
-        {
-          cmd = 'npm',
-          info = 'Installing NPM dependencies of ' .. lang .. ' parser',
-          err = 'Error during `npm install` (required for parser generation of '
-            .. lang
-            .. ' with npm dependencies)',
-          opts = {
-            args = { 'install' },
-            cwd = compile_location,
-          },
-        },
-      })
-    end
-    vim.list_extend(command_list, {
-      {
-        cmd = vim.fn.exepath('tree-sitter'),
-        info = 'Generating source files from grammar.js...',
-        err = 'Error during "tree-sitter generate"',
-        opts = {
-          args = M.ts_generate_args,
-          cwd = compile_location,
-        },
-      },
-    })
+    util.delete(fs.joinpath(cache_dir, project_name))
   end
-  vim.list_extend(command_list, {
-    shell.select_compile_command(repo, cc, compile_location),
-    {
-      cmd = function()
-        uv.fs_copyfile(fs.joinpath(compile_location, 'parser.so'), parser_lib_name)
-      end,
-    },
-    {
-      cmd = function()
-        local file = assert(
-          io.open(
-            fs.joinpath(config.get_install_dir('parser-info') or '', lang .. '.revision'),
-            'w'
-          )
-        )
-        file:write(revision or '')
-        file:close()
-      end,
-    },
-  })
-  if not from_local_path then
-    vim.list_extend(command_list, {
-      {
-        cmd = function()
-          vim.fn.delete(fs.joinpath(cache_dir, project_name), 'rf')
-        end,
-      },
-    })
+
+  local queries = fs.joinpath(config.get_install_dir('queries'), lang)
+  local queries_src = M.get_package_path('runtime', 'queries', lang)
+  uv_unlink(queries)
+  err = uv_symlink(queries_src, queries, { dir = true, junction = true })
+  a.main()
+  if err then
+    logger:error(err)
   end
+  logger:info('Parser installed')
+end
 
-  if with_sync then
-    if iter_cmd_sync(command_list) == true then
-      vim.notify('Parser for ' .. lang .. ' has been installed')
+--- Throttles a function using the first argument as an ID
+---
+--- If function is already running then the function will be scheduled to run
+--- again once the running call has finished.
+---
+---   fn#1        _/‾\__/‾\_/‾\_____________________________
+---   throttled#1 _/‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾\/‾‾‾‾‾‾‾‾‾‾\____________
+--
+---   fn#2        ______/‾\___________/‾\___________________
+---   throttled#2 ______/‾‾‾‾‾‾‾‾‾‾\__/‾‾‾‾‾‾‾‾‾‾\__________
+---
+---
+--- @generic F: function
+--- @param fn F Function to throttle
+--- @return F throttled function.
+local function throttle_by_id(fn)
+  local scheduled = {} --- @type table<any,boolean>
+  local running = {} --- @type table<any,boolean>
+  return function(id, ...)
+    if scheduled[id] then
+      -- If fn is already scheduled, then drop
+      return
+    end
+    if not running[id] then
+      scheduled[id] = true
+    end
+    if running[id] then
+      return
+    end
+    while scheduled[id] do
+      scheduled[id] = nil
+      running[id] = true
+      fn(id, ...)
+      running[id] = nil
     end
-  else
-    iter_cmd(command_list, 1, lang, 'Parser for ' .. lang .. ' has been installed')
   end
 end
 
+-- Async functions must not be interleaved
+local install_lang_throttled = throttle_by_id(install_lang)
+
 ---@class InstallOptions
----@field with_sync boolean
 ---@field force boolean
 ---@field generate_from_grammar boolean
 ---@field skip table
 
--- Install a parser
----@param languages? string[]|string
----@param options? InstallOptions
-function M.install(languages, options)
+--- Install a parser
+--- @param languages? string[]|string
+--- @param options? InstallOptions
+--- @param _callback? fun()
+local function install(languages, options, _callback)
   options = options or {}
-  local with_sync = options.with_sync
   local force = options.force
   local generate_from_grammar = options.generate_from_grammar
   local skip = options.skip
 
-  reset_progress_counter()
-
   if vim.fn.executable('git') == 0 then
-    api.nvim_err_writeln('Git is required on your system to run this command')
+    log.error('Git is required on your system to run this command')
     return
   end
 
   local cache_dir = vim.fn.stdpath('cache')
   local install_dir = config.get_install_dir('parser')
 
-  if languages == 'all' then
+  if not languages or type(languages) == 'string' then
+    languages = { languages }
+  end
+
+  if languages[1] == 'all' then
     force = true
   end
 
   languages = config.norm_languages(languages, skip)
 
+  local tasks = {} --- @type fun()[]
+  local done = 0
   for _, lang in ipairs(languages) do
-    install_lang(lang, cache_dir, install_dir, force, with_sync, generate_from_grammar)
-    uv.fs_symlink(
-      shell.get_package_path('runtime', 'queries', lang),
-      fs.joinpath(config.get_install_dir('queries'), lang),
-      { dir = true, junction = true } -- needed on Windows (non-junction links require admin)
-    )
+    tasks[#tasks + 1] = a.sync(function()
+      a.main()
+      install_lang_throttled(lang, cache_dir, install_dir, force, generate_from_grammar)
+      done = done + 1
+    end)