refactor: rewrite installation using jobs and async

Replace sync variants with callback support

15 files changed, 946 insertions, 704 deletions

diff --git a/README.md b/README.md
index 635b1ed15..523940b63 100644
--- a/README.md
+++ b/README.md
@@ -59,9 +59,6 @@ require'nvim-treesitter'.setup {
   -- List of parsers to ignore installing
   ignore_install = { },
 
-  -- Install parsers synchronously (only applied to `ensure_install`)
-  sync_install = false,
-
   -- Automatically install missing parsers when entering buffer
   auto_install = false,
 
diff --git a/TODO.md b/TODO.md
index 9c2300eb1..d61d179d6 100644
--- a/TODO.md
+++ b/TODO.md
@@ -7,7 +7,6 @@ This document lists the planned and finished changes in this rewrite towards [Nv
 - [ ] **`query_predicates.lua`:** upstream/remove
 - [ ] **`parsers.lua`:** modularize?
 - [ ] **`parsers.lua`:** assign tiers
-- [ ] **`install.lua`:** fix messages, add sync support (@lewis6991)
 - [ ] **`install.lua`:** simplify compilation:
   - hardcode one compiler + args per platform
   - provide `install.compile_command` for overriding (function that takes files, ...?)
@@ -32,5 +31,6 @@ This document lists the planned and finished changes in this rewrite towards [Nv
 - [X] install parsers to standard directory by default
 - [X] remove bundled queries from runtimepath; copy on parser install
 - [X] general refactor and cleanup
+- [X] rewrite installation using async module (drop support for sync; use callback instead)
 - [X] switch to upstream injection format
 - [X] remove locals from highlighting (cf. https://github.com/nvim-treesitter/nvim-treesitter/issues/3944#issuecomment-1458782497)
diff --git a/doc/nvim-treesitter.txt b/doc/nvim-treesitter.txt
index 7054121b5..a7c6bfa69 100644
--- a/doc/nvim-treesitter.txt
+++ b/doc/nvim-treesitter.txt
@@ -38,16 +38,13 @@ To install supported parsers and queries, put this in your `init.lua` file:
 
 >lua
   require'nvim-treesitter.configs'.setup {
-    -- A directory to install the parsers into.
+    -- A directory to install the parsers and queries to.
     -- Defaults to the `stdpath('data')/site` dir.
     install_dir = "/some/path/to/store/parsers",
 
     -- A list of parser names, or "core", "stable", "community", "unstable"
     ensure_install = { "core", "rust" },
 
-    -- Install parsers synchronously (only applied to `ensure_installed`)
-    sync_install = false,
-
     -- Automatically install missing parsers when entering buffer
     auto_install = false,
 
@@ -66,10 +63,6 @@ COMMANDS                                            *nvim-treesitter-commands*
 Install one or more treesitter parsers.
 You can use |:TSInstall| `all` to install all parsers. Use |:TSInstall!| to
 force the reinstallation of already installed parsers.
-							      *:TSInstallSync*
-:TSInstallSync {language} ... ~
-
-Perform the |:TSInstall| operation synchronously.
 
 							      *:TSInstallInfo*
 :TSInstallInfo ~
@@ -83,11 +76,6 @@ Update the installed parser for one more {language} or all installed parsers
 if {language} is omitted. The specified parser is installed if it is not already
 installed.
 
-							       *:TSUpdateSync*
-:TSUpdateSync {language} ... ~
-
-Perform the |:TSUpdate| operation synchronously.
-
 								*:TSUninstall*
 :TSUninstall {language} ... ~
 
diff --git a/lua/nvim-treesitter/async.lua b/lua/nvim-treesitter/async.lua
new file mode 100644
index 000000000..57a670ef6
--- /dev/null
+++ b/lua/nvim-treesitter/async.lua
@@ -0,0 +1,110 @@
+local co = coroutine
+
+local M = {}
+
+---Executes a future with a callback when it is done
+--- @param func function
+--- @param callback function
+--- @param ... unknown
+local function execute(func, callback, ...)
+  local thread = co.create(func)
+
+  local function step(...)
+    local ret = { co.resume(thread, ...) }
+    --- @type boolean, any
+    local stat, nargs_or_err = unpack(ret)
+
+    if not stat then
+      error(
+        string.format(
+          'The coroutine failed with this message: %s\n%s',
+          nargs_or_err,
+          debug.traceback(thread)
+        )
+      )
+    end
+
+    if co.status(thread) == 'dead' then
+      if callback then
+        callback(unpack(ret, 3, table.maxn(ret)))
+      end
+      return
+    end
+
+    --- @type function, any[]
+    local fn, args = ret[3], { unpack(ret, 4, table.maxn(ret)) }
+    args[nargs_or_err] = step
+    fn(unpack(args, 1, nargs_or_err))
+  end
+
+  step(...)
+end
+
+--- Creates an async function with a callback style function.
+--- @generic F: function
+--- @param func F
+--- @param argc integer
+--- @return F
+function M.wrap(func, argc)
+  --- @param ... unknown
+  --- @return unknown
+  return function(...)
+    return co.yield(argc, func, ...)
+  end
+end
+
+---Use this to create a function which executes in an async context but
+---called from a non-async context. Inherently this cannot return anything
+---since it is non-blocking
+--- @generic F: function
+--- @param func async F
+--- @param nargs? integer
+--- @return F
+function M.sync(func, nargs)
+  nargs = nargs or 0
+  return function(...)
+    local callback = select(nargs + 1, ...)
+    execute(func, callback, unpack({ ... }, 1, nargs))
+  end
+end
+
+--- @param n integer max number of concurrent jobs
+--- @param interrupt_check? function
+--- @param thunks function[]
+--- @return any
+function M.join(n, interrupt_check, thunks)
+  return co.yield(1, function(finish)
+    if #thunks == 0 then
+      return finish()
+    end
+
+    local remaining = { select(n + 1, unpack(thunks)) }
+    local to_go = #thunks
+
+    local ret = {} --- @type any[]
+
+    local function cb(...)
+      ret[#ret + 1] = { ... }
+      to_go = to_go - 1
+      if to_go == 0 then
+        finish(ret)
+      elseif not interrupt_check or not interrupt_check() then
+        if #remaining > 0 then
+          local next_task = table.remove(remaining)
+          next_task(cb)
+        end
+      end
+    end
+
+    for i = 1, math.min(n, #thunks) do
+      thunks[i](cb)
+    end
+  end, 1)
+end
+
+---An async function that when called will yield to the Neovim scheduler to be
+---able to call the API.
+--- @type fun()
+M.main = M.wrap(vim.schedule, 1)
+
+return M
diff --git a/lua/nvim-treesitter/config.lua b/lua/nvim-treesitter/config.lua
index 01880a8bc..b8a9f6e91 100644
--- a/lua/nvim-treesitter/config.lua
+++ b/lua/nvim-treesitter/config.lua
@@ -1,7 +1,6 @@
 local M = {}
 
 ---@class TSConfig
----@field sync_install boolean
 ---@field auto_install boolean
 ---@field ensure_install string[]
 ---@field ignore_install string[]
@@ -9,7 +8,6 @@ local M = {}
 
 ---@type TSConfig
 local config = {
-  sync_install = false,
   auto_install = false,
   ensure_install = {},
   ignore_install = {},
@@ -38,7 +36,11 @@ function M.setup(user_data)
           and not vim.list_contains(M.installed_parsers(), lang)
           and not vim.list_contains(config.ignore_install, lang)
         then
-          require('nvim-treesitter.install').install(lang)
+          require('nvim-treesitter.install').install(lang, nil, function()
+            -- Need to pcall since 'FileType' can be triggered multiple times
+            -- per buffer
+            pcall(vim.treesitter.start, args.buf, lang)
+          end)
         end
       end,
     })
@@ -48,9 +50,7 @@ function M.setup(user_data)
     local to_install = M.norm_languages(config.ensure_install, { ignored = true, installed = true })
 
     if #to_install > 0 then
-      require('nvim-treesitter.install').install(to_install, {
-        with_sync = config.sync_install,
-      })
+      require('nvim-treesitter.install').install(to_install)
     end
   end
 end
@@ -63,9 +63,10 @@ function M.get_install_dir(dir_name)
   local dir = vim.fs.joinpath(config.install_dir, dir_name)
 
   if not vim.loop.fs_stat(dir) then
-    local ok, error = pcall(vim.fn.mkdir, dir, 'p', '0755')
+    local ok, err = pcall(vim.fn.mkdir, dir, 'p', '0755')
     if not ok then
-      vim.notify(error, vim.log.levels.ERROR)
+      local log = require('nvim-treesitter.log')
+      log.error(err --[[@as string]])
     end
   end
   return dir
@@ -105,35 +106,37 @@ function M.norm_languages(languages, skip)
     end
     languages = parsers.get_available()
   end
+  --TODO(clason): skip and warn on unavailable parser
 
   for i, tier in ipairs(parsers.tiers) do
     if vim.list_contains(languages, tier) then
       languages = vim.iter.filter(function(l)
         return l ~= tier
-      end, languages)
+      end, languages) --[[@as string[] ]]
       vim.list_extend(languages, parsers.get_available(i))
     end
   end
 
+  --TODO(clason): support skipping tiers
   if skip and skip.ignored then
     local ignored = config.ignore_install
     languages = vim.iter.filter(function(v)
       return not vim.list_contains(ignored, v)
-    end, languages)
+    end, languages) --[[@as string[] ]]
   end
 
   if skip and skip.installed then
     local installed = M.installed_parsers()
     languages = vim.iter.filter(function(v)
       return not vim.list_contains(installed, v)
-    end, languages)
+    end, languages) --[[@as string[] ]]
   end
 
   if skip and skip.missing then
     local installed = M.installed_parsers()
     languages = vim.iter.filter(function(v)
       return vim.list_contains(installed, v)
-    end, languages)
+    end, languages) --[[@as string[] ]]
   end
 
   return languages
diff --git a/lua/nvim-treesitter/health.lua b/lua/nvim-treesitter/health.lua
index e8c945f6c..1e17bb00b 100644
--- a/lua/nvim-treesitter/health.lua
+++ b/lua/nvim-treesitter/health.lua
@@ -1,6 +1,6 @@
-local shell = require('nvim-treesitter.shell_cmds')
 local install = require('nvim-treesitter.install')
 local config = require('nvim-treesitter.config')
+local util = require('nvim-treesitter.util')
 local tsq = vim.treesitter.query
 
 local M = {}
@@ -62,7 +62,7 @@ local function install_health()
     vim.health.ok('`git` executable found.')
   end
 
-  local cc = shell.select_executable(install.compilers)
+  local cc = install.select_executable(install.compilers)
   if not cc then
     vim.health.error('`cc` executable not found.', {
       'Check that any of '
@@ -118,6 +118,7 @@ local function query_status(lang, query_group)
 end
 
 function M.check()
+  --- @type {[1]: string, [2]: string, [3]: string}[]
   local error_collection = {}
   -- Installation dependency checks
   install_health()
@@ -144,22 +145,19 @@ function M.check()
   if #error_collection > 0 then
     vim.health.start('The following errors have been detected:')
     for _, p in ipairs(error_collection) do
-      local lang, type, err = unpack(p)
+      local lang, type, err = p[1], p[2], p[3]
       local lines = {}
       table.insert(lines, lang .. '(' .. type .. '): ' .. err)
       local files = tsq.get_files(lang, type)
       if #files > 0 then
         table.insert(lines, lang .. '(' .. type .. ') is concatenated from the following files:')
         for _, file in ipairs(files) do
-          local fd = io.open(file, 'r')
-          if fd then
-            local ok, file_err = pcall(tsq.parse, lang, fd:read('*a'))
-            if ok then
-              table.insert(lines, '|  [OK]:"' .. file .. '"')
-            else
-              table.insert(lines, '| [ERR]:"' .. file .. '", failed to load: ' .. file_err)
-            end
-            fd:close()
+          local query = util.read_file(file)
+          local ok, file_err = pcall(tsq.parse, lang, query)
+          if ok then
+            table.insert(lines, '|  [OK]:"' .. file .. '"')
+          else
+            table.insert(lines, '| [ERR]:"' .. file .. '", failed to load: ' .. file_err)
           end
         end
       end
diff --git a/lua/nvim-treesitter/install.lua b/lua/nvim-treesitter/install.lua
index 0db61137c..ab9543673 100644
--- a/lua/nvim-treesitter/install.lua
+++ b/lua/nvim-treesitter/install.lua
@@ -2,194 +2,56 @@ local api = vim.api
 local fs = vim.fs
 local uv = vim.loop
 
-local parsers = require('nvim-treesitter.parsers')
+local a = require('nvim-treesitter.async')
 local config = require('nvim-treesitter.config')
-local shell = require('nvim-treesitter.shell_cmds')
-
-local M = {}
-
----@class LockfileInfo
----@field revision string
-
----@type table<string, LockfileInfo>
-local lockfile = {}
-
-M.compilers = { uv.os_getenv('CC'), 'cc', 'gcc', 'clang', 'cl', 'zig' }
-M.prefer_git = uv.os_uname().sysname == 'Windows_NT'
-M.command_extra_args = {}
-M.ts_generate_args = nil
-
-local started_commands = 0
-local finished_commands = 0
-local failed_commands = 0
-local stdout_output = {}
-local stderr_output = {}
-
----
---- JOB API functions
----
+local job = require('nvim-treesitter.job')
+local log = require('nvim-treesitter.log')
+local parsers = require('nvim-treesitter.parsers')
+local util = require('nvim-treesitter.util')
 
-local function reset_progress_counter()
-  if started_commands ~= finished_commands then
-    return
-  end
-  started_commands = 0
-  finished_commands = 0
-  failed_commands = 0
-  stdout_output = {}
-  stderr_output = {}
-end
+--- @type fun(path: string, new_path: string, flags?: table): string?
+local uv_copyfile = a.wrap(uv.fs_copyfile, 4)
 
-local function get_job_status()
-  return '[nvim-treesitter] ['
-    .. finished_commands
-    .. '/'
-    .. started_commands
-    .. (failed_commands > 0 and ', failed: ' .. failed_commands or '')
-    .. ']'
-end
+--- @type fun(path: string, mode: integer): string?
+local uv_mkdir = a.wrap(uv.fs_mkdir, 3)
 
----@param cmd Command
----@return string command
-local function get_command(cmd)
-  local options = ''
-  if cmd.opts and cmd.opts.args then
-    if M.command_extra_args[cmd.cmd] then
-      vim.list_extend(cmd.opts.args, M.command_extra_args[cmd.cmd])
-    end
-    for _, opt in ipairs(cmd.opts.args) do
-      options = string.format('%s %s', options, opt)
-    end
-  end
+--- @type fun(path: string, new_path: string): string?
+local uv_rename = a.wrap(uv.fs_rename, 3)
 
-  local command = string.format('%s %s', cmd.cmd, options)
-  if cmd.opts and cmd.opts.cwd then
-    command = shell.make_directory_change_for_command(cmd.opts.cwd, command)
-  end
-  return command
-end
+--- @type fun(path: string, new_path: string, flags?: table): string?
+local uv_symlink = a.wrap(uv.fs_symlink, 4)
 
----@param cmd_list Command[]
----@return boolean
-local function iter_cmd_sync(cmd_list)
-  for _, cmd in ipairs(cmd_list) do
-    if cmd.info then
-      vim.notify(cmd.info)
-    end
+--- @type fun(path: string): string?
+local uv_unlink = a.wrap(uv.fs_unlink, 2)
 
-    if type(cmd.cmd) == 'function' then
-      cmd.cmd()
-    else
-      local ret = vim.fn.system(get_command(cmd))
-      if vim.v.shell_error ~= 0 then
-        vim.notify(ret)
-        api.nvim_err_writeln(
-          (cmd.err and cmd.err .. '\n' or '')
-            .. 'Failed to execute the following command:\n'
-            .. vim.inspect(cmd)
-        )
-        return false
-      end
-    end
-  end
-
-  return true
-end
+local M = {}
 
-local function iter_cmd(cmd_list, i, lang, success_message)
-  if i == 1 then
-    started_commands = started_commands + 1
-  end
-  if i == #cmd_list + 1 then
-    finished_commands = finished_commands + 1
-    return vim.notify(get_job_status() .. ' ' .. success_message)
-  end
+---@class LockfileInfo
+---@field revision string
 
-  local attr = cmd_list[i]
-  if attr.info then
-    vim.notify(get_job_status() .. ' ' .. attr.info)
-  end
+---@type table<string, LockfileInfo>
+local lockfile = {}
 
-  if attr.opts and attr.opts.args and M.command_extra_args[attr.cmd] then
-    vim.list_extend(attr.opts.args, M.command_extra_args[attr.cmd])
-  end
+local max_jobs = 50
 
-  if type(attr.cmd) == 'function' then
-    local ok, err = pcall(attr.cmd)
-    if ok then
-      iter_cmd(cmd_list, i + 1, lang, success_message)
-    else
-      failed_commands = failed_commands + 1
-      finished_commands = finished_commands + 1
-      return api.nvim_err_writeln(
-        (attr.err or ('Failed to execute the following command:\n' .. vim.inspect(attr)))
-          .. '\n'
-          .. vim.inspect(err)
-      )
-    end
-  else
-    local handle
-    local stdout = uv.new_pipe(false)
-    local stderr = uv.new_pipe(false)
-    attr.opts.stdio = { nil, stdout, stderr }
-    ---@type userdata
-    handle = uv.spawn(
-      attr.cmd,
-      attr.opts,
-      vim.schedule_wrap(function(code)
-        if code ~= 0 then
-          stdout:read_stop()
-          stderr:read_stop()
-        end
-        stdout:close()
-        stderr:close()
-        handle:close()
-        if code ~= 0 then
-          failed_commands = failed_commands + 1
-          finished_commands = finished_commands + 1
-          if stdout_output[handle] and stdout_output[handle] ~= '' then
-            vim.notify(stdout_output[handle])
-          end
+local iswin = uv.os_uname().sysname == 'Windows_NT'
+local ismac = uv.os_uname().sysname == 'Darwin'
 
-          local err_msg = stderr_output[handle] or ''
-          api.nvim_err_writeln(
-            'nvim-treesitter['
-              .. lang
-              .. ']: '
-              .. (attr.err or ('Failed to execute the following command:\n' .. vim.inspect(attr)))
-              .. '\n'
-              .. err_msg
-          )
-          return
-        end
-        iter_cmd(cmd_list, i + 1, lang, success_message)
-      end)
-    )
-    uv.read_start(stdout, function(_, data)
-      if data then
-        stdout_output[handle] = (stdout_output[handle] or '') .. data
-      end
-    end)
-    uv.read_start(stderr, function(_, data)
-      if data then
-        stderr_output[handle] = (stderr_output[handle] or '') .. data
-      end
-    end)
-  end
-end
+--- @diagnostic disable-next-line:missing-parameter
+M.compilers = { uv.os_getenv('CC'), 'cc', 'gcc', 'clang', 'cl', 'zig' }
 
 ---
 --- PARSER INFO
 ---
 
 ---@param lang string
----@param validate boolean|nil
+---@param validate? boolean
 ---@return InstallInfo
 local function get_parser_install_info(lang, validate)
   local parser_config = parsers.configs[lang]
 
   if not parser_config then
-    error('Parser not available for language "' .. lang .. '"')
+    log.error('Parser not available for language "' .. lang .. '"')
   end
 
   local install_info = parser_config.install_info
@@ -204,14 +66,18 @@ local function get_parser_install_info(lang, validate)
   return install_info
 end
 
+--- @param ... string
+--- @return string
+function M.get_package_path(...)
+  return fs.joinpath(vim.fn.fnamemodify(debug.getinfo(1, 'S').source:sub(2), ':p:h:h:h'), ...)
+end
+
 ---@param lang string
----@return string|nil
-local function get_revision(lang)
+---@return string?
+local function get_target_revision(lang)
   if #lockfile == 0 then
-    local filename = shell.get_package_path('lockfile.json')
-    local file = assert(io.open(filename, 'r'))
-    lockfile = vim.json.decode(file:read('*all'))
-    file:close()
+    local filename = M.get_package_path('lockfile.json')
+    lockfile = vim.json.decode(util.read_file(filename)) --[[@as table<string, LockfileInfo>]]
   end
 
   local install_info = get_parser_install_info(lang)
@@ -225,19 +91,16 @@ local function get_revision(lang)
 end
 
 ---@param lang string
----@return string|nil
+---@return string?
 local function get_installed_revision(lang)
   local lang_file = fs.joinpath(config.get_install_dir('parser-info'), lang .. '.revision')
-  local file = assert(io.open(lang_file, 'r'))
-  local revision = file:read('*a')
-  file:close()
-  return revision
+  return util.read_file(lang_file)
 end
 
 ---@param lang string
 ---@return boolean
 local function needs_update(lang)
-  local revision = get_revision(lang)
+  local revision = get_target_revision(lang)
   return not revision or revision ~= get_installed_revision(lang)
 end
 
@@ -254,7 +117,7 @@ function M.info()
 
   for _, lang in pairs(parser_list) do
     local parser = (lang .. string.rep(' ', max_len - #lang + 1))
-    local output
+    local output --- @type string[]
     if vim.list_contains(installed, lang) then
       output = { parser .. '[✓] installed', 'DiagnosticOk' }
     elseif #api.nvim_get_runtime_file('parser/' .. lang .. '.*', true) > 0 then
@@ -270,13 +133,311 @@ end
 --- PARSER MANAGEMENT FUNCTIONS
 ---
 
+--- @param repo InstallInfo
+--- @param project_name string
+--- @param cache_dir string
+--- @param from_local_path boolean
+--- @return string
+local function get_compile_location(repo, cache_dir, project_name, from_local_path)
+  ---@type string compile_location only needed for typescript installs.
+  if from_local_path then
+    local compile_location = repo.url
+    if repo.location then
+      compile_location = fs.joinpath(compile_location, repo.location)
+    end
+    return compile_location
+  end
+
+  local repo_location = project_name
+  if repo.location then
+    repo_location = fs.joinpath(repo_location, repo.location)
+  end
+  return fs.joinpath(cache_dir, repo_location)
+end
+
+local function cc_err()
+  log.error('No C compiler found! "' .. table.concat(
+    vim.tbl_filter(
+      ---@param c string
+      ---@return boolean
+      function(c)
+        return type(c) == 'string'
+      end,
+      M.compilers
+    ),
+    '", "'
+  ) .. '" are not executable.')
+end
+
+--- @param logger Logger
+--- @param repo InstallInfo
+--- @param compile_location string
+local function do_generate_from_grammar(logger, repo, compile_location)
+  if repo.generate_requires_npm then
+    if vim.fn.executable('npm') ~= 1 then
+      logger:error('NPM requires to be installed from grammar.js')
+    end
+
+    logger:info('Installing NPM dependencies')
+    local r = job.run({ 'npm', 'install' }, { cwd = compile_location })
+    a.main()
+    if r.exit_code > 0 then
+      logger:error('Error during `npm install`')
+    end
+  end
+
+  logger:info('Generating source files from grammar.js...')
+
+  local r = job.run({
+    vim.fn.exepath('tree-sitter'),
+    'generate',
+    '--no-bindings',
+    '--abi',
+    tostring(vim.treesitter.language_version),
+  }, { cwd = compile_location })
+  a.main()
+  if r.exit_code > 0 then
+    logger:error('Error during "tree-sitter generate"')
+  end
+end
+
+---@param logger Logger
+---@param repo InstallInfo
+---@param project_name string
+---@param cache_dir string
+---@param revision string
+---@param project_dir string
+local function do_download_tar(logger, repo, project_name, cache_dir, revision, project_dir)
+  local is_github = repo.url:find('github.com', 1, true)
+  local url = repo.url:gsub('.git$', '')
+
+  local dir_rev = revision
+  if is_github and revision:find('^v%d') then
+    dir_rev = revision:sub(2)
+  end
+
+  local temp_dir = project_dir .. '-tmp'
+
+  util.delete(temp_dir)
+
+  logger:info('Downloading ' .. project_name .. '...')
+  local target = is_github and url .. '/archive/' .. revision .. '.tar.gz'
+    or url .. '/-/archive/' .. revision .. '/' .. project_name .. '-' .. revision .. '.tar.gz'
+
+  local r = job.run({
+    'curl',
+    '--silent',
+    '--show-error',
+    '-L', -- follow redirects
+    target,
+    '--output',
+    project_name .. '.tar.gz',
+  }, {
+    cwd = cache_dir,
+  })
+  a.main()
+  if r.exit_code > 0 then
+    logger:error(
+      'Error during download, please verify your internet connection: ' .. vim.inspect(r.stderr)
+    )
+  end
+
+  logger:debug('Creating temporary directory: ' .. temp_dir)
+  --TODO(clason): use vim.fn.mkdir(temp_dir, 'p') in case stdpath('cache') is not created
+  local err = uv_mkdir(temp_dir, 493)
+  a.main()
+  if err then
+    logger:error('Could not create %s-tmp: %s', project_name, err)
+  end
+
+  logger:info('Extracting ' .. project_name .. '...')
+  r = job.run({
+    'tar',
+    '-xzf',
+    project_name .. '.tar.gz',
+    '-C',
+    project_name .. '-tmp',
+  }, {
+    cwd = cache_dir,
+  })
+
+  a.main()
+  if r.exit_code > 0 then
+    logger:error('Error during tarball extraction: ' .. vim.inspect(r.stderr))
+  end
+
+  err = uv_unlink(project_dir .. '.tar.gz')
+  if err then
+    logger:error('Could not remove tarball: ' .. err)
+  end
+  a.main()
+
+  err = uv_rename(fs.joinpath(temp_dir, url:match('[^/]-$') .. '-' .. dir_rev), project_dir)
+  a.main()
+
+  if err then
+    logger:error('Could not rename temp: ' .. err)
+  end
+
+  util.delete(temp_dir)
+end
+
+---@param logger Logger
+---@param repo InstallInfo
+---@param project_name string
+---@param cache_dir string
+---@param revision string
+---@param project_dir string
+local function do_download_git(logger, repo, project_name, cache_dir, revision, project_dir)
+  logger:info('Downloading ' .. project_name .. '...')
+
+  local r = job.run({
+    'git',
+    'clone',
+    '--filter=blob:none',
+    repo.url,
+    project_name,
+  }, {
+    cwd = cache_dir,
+  })
+
+  a.main()
+
+  if r.exit_code > 0 then
+    logger:error(
+      'Error during download, please verify your internet connection: ' .. vim.inspect(r.stderr)
+    )
+  end
+
+  logger:info('Checking out locked revision')
+  r = job.run({
+    'git',
+    'checkout',
+    revision,
+  }, {
+    cwd = project_dir,
+  })
+
+  a.main()
+
+  if r.exit_code > 0 then
+    logger:error('Error while checking out revision: ' .. vim.inspect(r.stderr))
+  end
+end
+
+---@param executables string[]
+---@return string?
+function M.select_executable(executables)
+  return vim.tbl_filter(
+    ---@param c string
+    ---@return boolean
+    function(c)
+      return c ~= vim.NIL and vim.fn.executable(c) == 1
+    end,
+    executables
+  )[1]
+end
+
+-- Returns the compiler arguments based on the compiler and OS
+---@param repo InstallInfo
+---@param compiler string
+---@return string[]
+local function select_compiler_args(repo, compiler)
+  if compiler:find('cl$') or compiler:find('cl.exe$') then
+    return {
+      '/Fe:',
+      'parser.so',
+      '/Isrc',
+      repo.files,
+      '-Os',
+      '/utf-8',
+      '/LD',
+    }
+  end
+
+  if compiler:find('zig$') or compiler:find('zig.exe$') then
+    return {
+      'c++',
+      '-o',
+      'parser.so',
+      repo.files,
+      '-lc',
+      '-Isrc',
+      '-shared',
+      '-Os',
+    }
+  end
+
+  local args = {
+    '-o',
+    'parser.so',
+    '-I./src',
+    repo.files,
+    '-Os',
+    ismac and '-bundle' or '-shared',
+  }
+
+  if
+    #vim.tbl_filter(
+      --- @param file string
+      --- @return boolean
+      function(file)
+        local ext = vim.fn.fnamemodify(file, ':e')
+        return ext == 'cc' or ext == 'cpp' or ext == 'cxx'
+      end,
+      repo.files
+    ) > 0
+  then
+    table.insert(args, '-lstdc++')
+  end
+
+  if not iswin then
+    table.insert(args, '-fPIC')
+  end
+
+  return args
+end
+
+---@param repo InstallInfo
+---@return boolean
+local function can_download_tar(repo)
+  local can_use_tar = vim.fn.executable('tar') == 1 and vim.fn.executable('curl') == 1
+  local is_github = repo.url:find('github.com', 1, true)
+  local is_gitlab = repo.url:find('gitlab.com', 1, true)
+  return can_use_tar and (is_github or is_gitlab) and not iswin
+end
+
+-- Returns the compile command based on the OS and user options
+---@param repo InstallInfo
+---@param cc string
+---@param compile_location string
+---@return JobResult
+local function do_compile(repo, cc, compile_location)
+  local make = M.select_executable({ 'gmake', 'make' })
+
+  local cmd --- @type string[]
+  if cc:find('cl$') or cc:find('cl.exe$') or not repo.use_makefile or iswin or not make then
+    local args = vim.tbl_flatten(select_compiler_args(repo, cc))
+    cmd = vim.list_extend({ cc }, args)
+  else
+    cmd = {
+      make,
+      '--makefile=' .. M.get_package_path('scripts', 'compile_parsers.makefile'),
+      'CC=' .. cc,
+    }
+  end
+
+  local r = job.run(cmd, { cwd = compile_location })
+  a.main()
+  return r
+end
+
 ---@param lang string
 ---@param cache_dir string
 ---@param install_dir string
 ---@param force boolean
----@param with_sync boolean
 ---@param generate_from_grammar boolean
-local function install_lang(lang, cache_dir, install_dir, force, with_sync, generate_from_grammar)
+local function install_lang(lang, cache_dir, install_dir, force, generate_from_grammar)
   if vim.list_contains(config.installed_parsers(), lang) then
     if not force then
       local yesno =
@@ -288,278 +449,260 @@ local function install_lang(lang, cache_dir, install_dir, force, with_sync, gene
     end
   end
 
+  local cc = M.select_executable(M.compilers)
+  if not cc then
+    cc_err()
+    return
+  end
+
   local repo = get_parser_install_info(lang)
 
   local project_name = 'tree-sitter-' .. lang
+
+  local logger = log.new('install/' .. lang)
+
+  generate_from_grammar = repo.requires_generate_from_grammar or generate_from_grammar
+
+  if generate_from_grammar and vim.fn.executable('tree-sitter') ~= 1 then
+    logger:error('tree-sitter CLI not found: `tree-sitter` is not executable')
+  end
+
+  if generate_from_grammar and vim.fn.executable('node') ~= 1 then
+    logger:error('Node JS not found: `node` is not executable')
+  end
+
+  local revision = repo.revision or get_target_revision(lang)
+
   local maybe_local_path = fs.normalize(repo.url)
   local from_local_path = vim.fn.isdirectory(maybe_local_path) == 1
   if from_local_path then
     repo.url = maybe_local_path
   end
 
-  ---@type string compile_location only needed for typescript installs.
-  local compile_location
-  if from_local_path then
-    compile_location = repo.url
-    if repo.location then
-      compile_location = fs.joinpath(compile_location, repo.location)
-    end
-  else
-    local repo_location = project_name
-    if repo.location then
-      repo_location = fs.joinpath(repo_location, repo.location)
-    end
-    compile_location = fs.joinpath(cache_dir, repo_location)
-  end
-  local parser_lib_name = fs.joinpath(install_dir, lang) .. '.so'
+  if not from_local_path then
+    util.delete(fs.joinpath(cache_dir, project_name))
+    local project_dir = fs.joinpath(cache_dir, project_name)
 
-  generate_from_grammar = repo.requires_generate_from_grammar or generate_from_grammar
+    revision = revision or repo.branch or 'master'
 
-  if generate_from_grammar and vim.fn.executable('tree-sitter') ~= 1 then
-    api.nvim_err_writeln('tree-sitter CLI not found: `tree-sitter` is not executable')
-    if repo.requires_generate_from_grammar then
-      api.nvim_err_writeln(
-        'tree-sitter CLI is needed because the parser for `'
-          .. lang
-          .. '` needs to be generated from grammar'
-      )
-    end
-    return
-  else
-    if not M.ts_generate_args then
-      M.ts_generate_args = { 'generate', '--no-bindings', '--abi', vim.treesitter.language_version }
+    if can_download_tar(repo) then
+      do_download_tar(logger, repo, project_name, cache_dir, revision, project_dir)
+    else
+      do_download_git(logger, repo, project_name, cache_dir, revision, project_dir)
     end
   end
-  if generate_from_grammar and vim.fn.executable('node') ~= 1 then
-    api.nvim_err_writeln('Node JS not found: `node` is not executable')
-    return
-  end
-  local cc = shell.select_executable(M.compilers)
-  if not cc then
-    api.nvim_err_writeln(
-      'No C compiler found! "'
-        .. table.concat(
-          vim.tbl_filter(function(c) ---@param c string
-            return type(c) == 'string'
-          end, M.compilers),
-          '", "'
-        )
-        .. '" are not executable.'
-    )
-    return
+
+  local compile_location = get_compile_location(repo, cache_dir, project_name, from_local_path)
+
+  if generate_from_grammar then
+    do_generate_from_grammar(logger, repo, compile_location)
   end
 
-  local revision = repo.revision
-  if not revision then
-    revision = get_revision(lang)
+  logger:info('Compiling parser')
+  local r = do_compile(repo, cc, compile_location)
+  if r.exit_code > 0 then
+    logger:error('Error during compilation: ' .. vim.inspect(r.stderr))
   end
 
-  ---@class Command
-  ---@field cmd string
-  ---@field info string
-  ---@field err string
-  ---@field opts CmdOpts
+  local parser_lib_name = fs.joinpath(install_dir, lang) .. '.so'
+
+  local err = uv_copyfile(fs.joinpath(compile_location, 'parser.so'), parser_lib_name)
+  a.main()
+  if err then
+    logger:error(err)
+  end
 
-  ---@class CmdOpts
-  ---@field args string[]
-  ---@field cwd string
+  local revfile = fs.joinpath(config.get_install_dir('parser-info') or '', lang .. '.revision')
+  util.write_file(revfile, revision or '')
 
-  ---@type Command[]
-  local command_list = {}
   if not from_local_path then
-    vim.list_extend(command_list, {
-      {
-        cmd = function()
-          vim.fn.delete(fs.joinpath(cache_dir, project_name), 'rf')
-        end,
-      },
-    })
-    vim.list_extend(
-      command_list,
-      shell.select_download_commands(repo, project_name, cache_dir, revision, M.prefer_git)
-    )
-  end
-  if generate_from_grammar then
-    if repo.generate_requires_npm then
-      if vim.fn.executable('npm') ~= 1 then
-        api.nvim_err_writeln('`' .. lang .. '` requires NPM to be installed from grammar.js')
-        return
-      end
-      vim.list_extend(command_list, {
-        {
-          cmd = 'npm',
-          info = 'Installing NPM dependencies of ' .. lang .. ' parser',
-          err = 'Error during `npm install` (required for parser generation of '
-            .. lang
-            .. ' with npm dependencies)',
-          opts = {
-            args = { 'install' },
-            cwd = compile_location,
-          },
-        },
-      })
-    end
-    vim.list_extend(command_list, {
-      {
-        cmd = vim.fn.exepath('tree-sitter'),
-        info = 'Generating source files from grammar.js...',
-        err = 'Error during "tree-sitter generate"',
-        opts = {
-          args = M.ts_generate_args,
-          cwd = compile_location,
-        },
-      },
-    })
+    util.delete(fs.joinpath(cache_dir, project_name))
   end
-  vim.list_extend(command_list, {
-    shell.select_compile_command(repo, cc, compile_location),
-    {
-      cmd = function()
-        uv.fs_copyfile(fs.joinpath(compile_location, 'parser.so'), parser_lib_name)
-      end,
-    },
-    {
-      cmd = function()
-        local file = assert(
-          io.open(
-            fs.joinpath(config.get_install_dir('parser-info') or '', lang .. '.revision'),
-            'w'
-          )
-        )
-        file:write(revision or '')
-        file:close()
-      end,
-    },
-  })
-  if not from_local_path then
-    vim.list_extend(command_list, {
-      {
-        cmd = function()
-          vim.fn.delete(fs.joinpath(cache_dir, project_name), 'rf')
-        end,
-      },
-    })
+
+  local queries = fs.joinpath(config.get_install_dir('queries'), lang)
+  local queries_src = M.get_package_path('runtime', 'queries', lang)
+  uv_unlink(queries)
+  err = uv_symlink(queries_src, queries, { dir = true, junction = true })
+  a.main()
+  if err then
+    logger:error(err)
   end
+  logger:info('Parser installed')
+end
 
-  if with_sync then
-    if iter_cmd_sync(command_list) == true then
-      vim.notify('Parser for ' .. lang .. ' has been installed')
+--- Throttles a function using the first argument as an ID
+---
+--- If function is already running then the function will be scheduled to run
+--- again once the running call has finished.
+---
+---   fn#1        _/‾\__/‾\_/‾\_____________________________
+---   throttled#1 _/‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾\/‾‾‾‾‾‾‾‾‾‾\____________
+--
+---   fn#2        ______/‾\___________/‾\___________________
+---   throttled#2 ______/‾‾‾‾‾‾‾‾‾‾\__/‾‾‾‾‾‾‾‾‾‾\__________
+---
+---
+--- @generic F: function
+--- @param fn F Function to throttle
+--- @return F throttled function.
+local function throttle_by_id(fn)
+  local scheduled = {} --- @type table<any,boolean>
+  local running = {} --- @type table<any,boolean>
+  return function(id, ...)
+    if scheduled[id] then
+      -- If fn is already scheduled, then drop
+      return
+    end
+    if not running[id] then
+      scheduled[id] = true
+    end
+    if running[id] then
+      return
+    end
+    while scheduled[id] do
+      scheduled[id] = nil
+      running[id] = true
+      fn(id, ...)
+      running[id] = nil
     end
-  else
-    iter_cmd(command_list, 1, lang, 'Parser for ' .. lang .. ' has been installed')
   end
 end
 
+-- Async functions must not be interleaved
+local install_lang_throttled = throttle_by_id(install_lang)
+
 ---@class InstallOptions
----@field with_sync boolean
 ---@field force boolean
 ---@field generate_from_grammar boolean
 ---@field skip table
 
--- Install a parser
----@param languages? string[]|string
----@param options? InstallOptions
-function M.install(languages, options)
+--- Install a parser
+--- @param languages? string[]|string
+--- @param options? InstallOptions
+--- @param _callback? fun()
+local function install(languages, options, _callback)
   options = options or {}
-  local with_sync = options.with_sync
   local force = options.force
   local generate_from_grammar = options.generate_from_grammar
   local skip = options.skip
 
-  reset_progress_counter()
-
   if vim.fn.executable('git') == 0 then
-    api.nvim_err_writeln('Git is required on your system to run this command')
+    log.error('Git is required on your system to run this command')
     return
   end
 
   local cache_dir = vim.fn.stdpath('cache')
   local install_dir = config.get_install_dir('parser')
 
-  if languages == 'all' then
+  if not languages or type(languages) == 'string' then
+    languages = { languages }
+  end
+
+  if languages[1] == 'all' then
     force = true
   end
 
   languages = config.norm_languages(languages, skip)
 
+  local tasks = {} --- @type fun()[]
+  local done = 0
   for _, lang in ipairs(languages) do
-    install_lang(lang, cache_dir, install_dir, force, with_sync, generate_from_grammar)
-    uv.fs_symlink(
-      shell.get_package_path('runtime', 'queries', lang),
-      fs.joinpath(config.get_install_dir('queries'), lang),
-      { dir = true, junction = true } -- needed on Windows (non-junction links require admin)
-    )
+    tasks[#tasks + 1] = a.sync(function()
+      a.main()
+      install_lang_throttled(lang, cache_dir, install_dir, force, generate_from_grammar)
+      done = done + 1
+    end)
+  end
+
+  a.join(max_jobs, nil, tasks)
+  if #tasks > 1 then
+    a.main()
+    log.info('Installed %d/%d parsers', done, #tasks)
   end
 end
 
+M.install = a.sync(install, 2)
+
 ---@class UpdateOptions
----@field with_sync boolean
 
 ---@param languages? string[]|string
----@param options? UpdateOptions
-function M.update(languages, options)
-  options = options or {}
-
-  reset_progress_counter()
+---@param _options? UpdateOptions
+---@param _callback function
+M.update = a.sync(function(languages, _options, _callback)
   M.lockfile = {}
 
-  languages = config.norm_languages(languages or 'all', { ignored = true, missing = true })
+  if not languages or #languages == 0 then
+    languages = 'all'
+  end
+  languages = config.norm_languages(languages, { ignored = true, missing = true })
   languages = vim.iter.filter(needs_update, languages) --- @type string[]
 
   if #languages > 0 then
-    M.install(languages, {
-      force = true,
-      with_sync = options.with_sync,
-    })
+    install(languages, { force = true })
   else
-    vim.notify('All parsers are up-to-date')
+    log.info('All parsers are up-to-date')
   end
-end
+end, 2)
 
 --- @param lang string
 --- @param parser string
 --- @param queries string
-local function uninstall(lang, parser, queries)
+local function uninstall_lang(lang, parser, queries)
+  local logger = log.new('uninstall/' .. lang)
+  logger:debug('Uninstalling ' .. lang)
   if vim.fn.filereadable(parser) ~= 1 then
     return
   end
 
-  iter_cmd({
-    {
-      cmd = function()
-        uv.fs_unlink(parser)
-      end,
-    },
-    {
-      cmd = function()
-        uv.fs_unlink(queries)
-      end,
-    },
-  }, 1, lang, 'Parser for ' .. lang .. ' has been uninstalled')
+  logger:debug('Unlinking ' .. parser)
+  local perr = uv_unlink(parser)
+  a.main()
+
+  if perr then
+    log.error(perr)
+  end
+
+  logger:debug('Unlinking ' .. queries)
+  local qerr = uv_unlink(queries)
+  a.main()
+
+  if qerr then
+    logger:error(qerr)
+  end
+
+  logger:info('Parser uninstalled')
 end
 
 --- @param languages string[]|string
-function M.uninstall(languages)
-  reset_progress_counter()
-
+--- @param _options? UpdateOptions
+--- @param _callback fun()
+M.uninstall = a.sync(function(languages, _options, _callback)
   languages = config.norm_languages(languages or 'all', { missing = true })
 
   local parser_dir = config.get_install_dir('parser')
   local query_dir = config.get_install_dir('queries')
   local installed = config.installed_parsers()
 
+  local tasks = {} --- @type fun()[]
+  local done = 0
   for _, lang in ipairs(languages) do
     if not vim.list_contains(installed, lang) then
-      vim.notify(
-        'Parser for ' .. lang .. ' is is not managed by nvim-treesitter',
-        vim.log.levels.ERROR
-      )
+      log.warn('Parser for ' .. lang .. ' is is not managed by nvim-treesitter')
     else
       local parser = fs.joinpath(parser_dir, lang) .. '.so'
       local queries = fs.joinpath(query_dir, lang)
-      uninstall(lang, parser, queries)
+      tasks[#tasks + 1] = a.sync(function()
+        uninstall_lang(lang, parser, queries)
+        done = done + 1
+      end)
     end
   end
-end
+
+  a.join(max_jobs, nil, tasks)
+  if #tasks > 1 then
+    a.main()
+    log.info('Uninstalled %d/%d parsers', done, #tasks)
+  end
+end, 2)
 
 return M
diff --git a/lua/nvim-treesitter/job.lua b/lua/nvim-treesitter/job.lua
new file mode 100644
index 000000000..9c2788cb0
--- /dev/null
+++ b/lua/nvim-treesitter/job.lua
@@ -0,0 +1,122 @@
+-- Interface with Neovim job control and provide a simple job sequencing structure
+local uv = vim.loop
+local a = require('nvim-treesitter.async')
+local log = require('nvim-treesitter.log')
+
+local M = { JobResult = {}, Opts = {} }
+
+--- @class JobResult
+--- @field exit_code integer
+--- @field signal integer | string
+--- @field stdout string[]
+--- @field stderr string[]
+
+--- @class JobOpts
+--- @field cwd  string
+--- @field timeout  integer
+--- @field env  string[]
+--- @field on_stderr  fun(_: string)
+--- @field on_stdout  fun(_: string)
+
+--- Wrapper for vim.loop.spawn. Takes a command, options, and callback just like
+--- vim.loop.spawn, but ensures that all output from the command has been
+--- flushed before calling the callback.
+--- @param cmd string
+--- @param options uv.aliases.spawn_options
+--- @param callback fun(exit_code: integer, signal: integer|string)
+local function spawn(cmd, options, callback)
+  local handle --- @type uv_process_t?
+  local timer --- @type uv_timer_t
+  log.trace('running job: (cwd=%s) %s %s', options.cwd, cmd, table.concat(options.args, ' '))
+  handle = uv.spawn(cmd, options, function(exit_code, signal)
+    ---@cast handle -nil
+
+    handle:close()
+    if timer then
+      timer:stop()
+      timer:close()
+    end
+
+    callback(exit_code, signal)
+  end)
+
+  --- @type integer?
+  --- @diagnostic disable-next-line:undefined-field
+  local timeout = options.timeout
+
+  if timeout then
+    timer = assert(uv.new_timer())
+    timer:start(timeout, 0, function()
+      timer:stop()
+      timer:close()
+      if handle and handle:is_active() then
+        log.warn('Killing %s due to timeout!', cmd)
+        handle:kill('sigint')
+        handle:close()
+        for _, pipe in
+          pairs(options.stdio --[[@as uv_pipe_t[] ]])
+        do
+          pipe:close()
+        end
+        callback(-9999, 'sigint')
+      end
+    end)
+  end
+end
+
+--- Main exposed function for the jobs module. Takes a task and options and
+--- returns an async function that will run the task with the given opts via
+--- vim.loop.spawn
+--- @param task string[]
+--- @param opts JobOpts
+--- @param callback fun(_: JobResult)
+--- @type fun(task: string|string[], opts: JobOpts): JobResult
+M.run = a.wrap(function(task, opts, callback)
+  local stdout_data = {}
+  local stderr_data = {}
+
+  local stdout = assert(uv.new_pipe(false))
+  local stderr = assert(uv.new_pipe(false))
+
+  spawn(task[1], {
+    args = { unpack(task, 2) },
+    stdio = { nil, stdout, stderr },
+    cwd = opts.cwd,
+    timeout = opts.timeout and 1000 * opts.timeout or nil,
+    env = opts.env,
+    hide = true,
+  }, function(exit_code, signal)
+    callback({
+      exit_code = exit_code,
+      signal = signal,
+      stdout = stdout_data,
+      stderr = stderr_data,
+    })
+  end)
+
+  for kind, pipe in pairs({ stdout = stdout, stderr = stderr }) do
+    pipe:read_start(function(err, data)
+      if kind == 'stderr' and opts.on_stderr and data then
+        opts.on_stderr(data)
+      end
+      if kind == 'stdout' and opts.on_stdout and data then
+        opts.on_stdout(data)
+      end
+      if data then
+        log.trace('%s -> %s', kind, data)
+      end
+      if err then
+        log.error(err)
+      end
+      if data ~= nil then
+        local output = kind == 'stdout' and stdout_data or stderr_data
+        table.insert(output, vim.trim(data))
+      else
+        pipe:read_stop()
+        pipe:close()
+      end
+    end)
+  end
+end, 3)
+
+return M
diff --git a/lua/nvim-treesitter/log.lua b/lua/nvim-treesitter/log.lua
new file mode 100644
index 000000000..83f0b41c2
--- /dev/null
+++ b/lua/nvim-treesitter/log.lua
@@ -0,0 +1,133 @@
+local api = vim.api
+
+-- TODO(lewis6991): write these out to a file
+local messages = {} --- @type {[1]: string, [2]: string?, [3]: string}[]
+
+local sev_to_hl = {
+  trace = 'DiagnosticHint',
+  debug = 'Normal',
+  info = 'MoreMsg',
+  warn = 'WarningMsg',
+  error = 'ErrorMsg',
+}
+
+---@param ctx string?
+---@param m string
+---@param ... any
+local function trace(ctx, m, ...)
+  messages[#messages + 1] = { 'trace', ctx, string.format(m, ...) }
+end
+
+---@param ctx string?
+---@param m string
+---@param ... any
+local function debug(ctx, m, ...)
+  messages[#messages + 1] = { 'debug', ctx, string.format(m, ...) }
+end
+
+---@param ctx string?
+---@return string
+local function mkpfx(ctx)
+  return ctx and string.format('[nvim-treesitter/%s]', ctx) or '[nvim-treesitter]'
+end
+
+---@param ctx string?
+---@param m string
+---@param ... any
+local function info(ctx, m, ...)
+  local m1 = string.format(m, ...)
+  messages[#messages + 1] = { 'info', ctx, m1 }
+  api.nvim_echo({ { mkpfx(ctx) .. ': ' .. m1, sev_to_hl.info } }, true, {})
+end
+
+---@param ctx string?
+---@param m string
+---@param ... any
+local function warn(ctx, m, ...)
+  local m1 = string.format(m, ...)
+  messages[#messages + 1] = { 'warn', ctx, m1 }
+  api.nvim_echo({ { mkpfx(ctx) .. ' warning: ' .. m1, sev_to_hl.warn } }, true, {})
+end
+
+---@param ctx string?
+---@param m string
+---@param ... any
+local function lerror(ctx, m, ...)
+  local m1 = string.format(m, ...)
+  messages[#messages + 1] = { 'error', ctx, m1 }
+  error(mkpfx(ctx) .. ' error: ' .. m1)
+end
+
+--- @class NTSLogModule
+--- @field trace fun(fmt: string, ...: any)
+--- @field debug fun(fmt: string, ...: any)
+--- @field info fun(fmt: string, ...: any)
+--- @field warn fun(fmt: string, ...: any)
+--- @field error fun(fmt: string, ...: any)
+local M = {}
+
+--- @class Logger
+--- @field ctx? string
+local Logger = {}
+
+M.Logger = Logger
+
+--- @param ctx? string
+--- @return Logger
+function M.new(ctx)
+  return setmetatable({ ctx = ctx }, { __index = Logger })
+end
+
+---@param m string
+---@param ... any
+function Logger:trace(m, ...)
+  trace(self.ctx, m, ...)
+end
+
+---@param m string
+---@param ... any
+function Logger:debug(m, ...)
+  debug(self.ctx, m, ...)
+end
+
+---@param m string
+---@param ... any
+function Logger:info(m, ...)
+  info(self.ctx, m, ...)
+end
+
+---@param m string
+---@param ... any
+function Logger:warn(m, ...)
+  warn(self.ctx, m, ...)
+end
+
+---@param m string
+---@param ... any
+function Logger:error(m, ...)
+  lerror(self.ctx, m, ...)
+end
+
+local noctx_logger = M.new()
+
+setmetatable(M, {
+  __index = function(t, k)
+    --- @diagnostic disable-next-line:no-unknown
+    t[k] = function(...)
+      return noctx_logger[k](noctx_logger, ...)
+    end
+    return t[k]
+  end,
+})
+
+function M.show()
+  for _, l in ipairs(messages) do
+    local sev, ctx, msg = l[1], l[2], l[3]
+    local hl = sev_to_hl[sev]
+    local text = ctx and string.format('%s(%s): %s', sev, ctx, msg)
+      or string.format('%s: %s', sev, msg)
+    api.nvim_echo({ { text, hl } }, false, {})
+  end
+end
+
+return M
diff --git a/lua/nvim-treesitter/parsers.lua b/lua/nvim-treesitter/parsers.lua
index 69e9500fe..114ab40c6 100644
--- a/lua/nvim-treesitter/parsers.lua
+++ b/lua/nvim-treesitter/parsers.lua
@@ -2818,17 +2818,18 @@ M.configs = {
 ---@param tier integer? only get parsers of specified tier
 ---@return string[]
 function M.get_available(tier)
+  --- @type string[]
   local parsers = vim.tbl_keys(M.configs)
   table.sort(parsers)
   if tier then
     parsers = vim.iter.filter(function(p)
       return M.configs[p].tier == tier
-    end, parsers)
+    end, parsers) --[[@as string[] ]]
   end
   if vim.fn.executable('tree-sitter') == 0 or vim.fn.executable('node') == 0 then
     parsers = vim.iter.filter(function(p)
       return not M.configs[p].install_info.requires_generate_from_grammar
-    end, parsers)
+    end, parsers) --[[@as string[] ]]
   end
   return parsers
 end
diff --git a/lua/nvim-treesitter/shell_cmds.lua b/lua/nvim-treesitter/shell_cmds.lua
deleted file mode 100644
index 4c631512d..000000000
--- a/lua/nvim-treesitter/shell_cmds.lua
+++ /dev/null
@@ -1,258 +0,0 @@
-local uv = vim.loop
-
-local iswin = uv.os_uname().sysname == 'Windows_NT'
-
-local M = {}
-
----@param executables string[]
----@return string|nil
-function M.select_executable(executables)
-  return vim.tbl_filter(function(c) ---@param c string
-    return c ~= vim.NIL and vim.fn.executable(c) == 1
-  end, executables)[1]
-end
-
--- Returns the compiler arguments based on the compiler and OS
----@param repo InstallInfo
----@param compiler string
----@return string[]
-function M.select_compiler_args(repo, compiler)
-  if compiler:find('cl$') or compiler:find('cl.exe$') then
-    return {
-      '/Fe:',
-      'parser.so',
-      '/Isrc',
-      repo.files,
-      '-Os',
-      '/utf-8',
-      '/LD',
-    }
-  elseif compiler:find('zig$') or compiler:find('zig.exe$') then
-    return {
-      'c++',
-      '-o',
-      'parser.so',
-      repo.files,
-      '-lc',
-      '-Isrc',
-      '-shared',
-      '-Os',
-    }
-  else
-    local args = {
-      '-o',
-      'parser.so',
-      '-I./src',
-      repo.files,
-      '-Os',
-    }
-    if uv.os_uname().sysname == 'Darwin' then
-      table.insert(args, '-bundle')
-    else
-      table.insert(args, '-shared')
-    end
-    if
-      #vim.tbl_filter(function(file) ---@param file string
-        local ext = vim.fn.fnamemodify(file, ':e')
-        return ext == 'cc' or ext == 'cpp' or ext == 'cxx'
-      end, repo.files) > 0
-    then
-      table.insert(args, '-lstdc++')
-    end
-    if not iswin then
-      table.insert(args, '-fPIC')
-    end
-    return args
-  end
-end
-
--- Returns the compile command based on the OS and user options
----@param repo InstallInfo
----@param cc string
----@param compile_location string
----@return Command
-function M.select_compile_command(repo, cc, compile_location)
-  local make = M.select_executable({ 'gmake', 'make' })
-  if cc:find('cl$') or cc:find('cl.exe$') or not repo.use_makefile or iswin or not make then
-    return {
-      cmd = cc,
-      info = 'Compiling...',
-      err = 'Error during compilation',
-      opts = {
-        args = vim.tbl_flatten(M.select_compiler_args(repo, cc)),
-        cwd = compile_location,
-      },
-    }
-  else
-    return {
-      cmd = make,
-      info = 'Compiling...',
-      err = 'Error during compilation',
-      opts = {
-        args = {
-          '--makefile=' .. M.get_package_path('scripts', 'compile_parsers.makefile'),
-          'CC=' .. cc,
-        },
-        cwd = compile_location,
-      },
-    }
-  end
-end
-
----@param repo InstallInfo
----@param project_name string
----@param cache_dir string
----@param revision string|nil
----@param prefer_git boolean
----@return table
-function M.select_download_commands(repo, project_name, cache_dir, revision, prefer_git)
-  local can_use_tar = vim.fn.executable('tar') == 1 and vim.fn.executable('curl') == 1
-  local is_github = repo.url:find('github.com', 1, true)
-  local is_gitlab = repo.url:find('gitlab.com', 1, true)
-  local project_dir = vim.fs.joinpath(cache_dir, project_name)
-
-  revision = revision or repo.branch or 'master'
-
-  if can_use_tar and (is_github or is_gitlab) and not prefer_git then
-    local url = repo.url:gsub('.git$', '')
-
-    local dir_rev = revision
-    if is_github and revision:find('^v%d') then
-      dir_rev = revision:sub(2)
-    end
-
-    local temp_dir = project_dir .. '-tmp'
-
-    return {
-      {
-        cmd = function()
-          vim.fn.delete(temp_dir, 'rf')
-        end,
-      },
-      {
-        cmd = 'curl',
-        info = 'Downloading ' .. project_name .. '...',
-        err = 'Error during download, please verify your internet connection',
-        opts = {
-          args = {
-            '--silent',
-            '-L', -- follow redirects
-            is_github and url .. '/archive/' .. revision .. '.tar.gz'
-              or url
-                .. '/-/archive/'
-                .. revision
-                .. '/'
-                .. project_name
-                .. '-'
-                .. revision
-                .. '.tar.gz',
-            '--output',
-            project_name .. '.tar.gz',
-          },
-          cwd = cache_dir,
-        },
-      },
-      {
-        cmd = function()
-          --TODO(clason): use vim.fn.mkdir(temp_dir, 'p') in case stdpath('cache') is not created
-          uv.fs_mkdir(temp_dir, 493)
-        end,
-        info = 'Creating temporary directory',
-        err = 'Could not create ' .. project_name .. '-tmp',
-      },
-      {
-        cmd = 'tar',
-        info = 'Extracting ' .. project_name .. '...',
-        err = 'Error during tarball extraction.',
-        opts = {
-          args = {
-            '-xvzf',
-            project_name .. '.tar.gz',
-            '-C',
-            project_name .. '-tmp',
-          },
-          cwd = cache_dir,
-        },
-      },
-      {
-        cmd = function()
-          uv.fs_unlink(project_dir .. '.tar.gz')
-        end,
-      },
-      {
-        cmd = function()
-          uv.fs_rename(
-            vim.fs.joinpath(temp_dir, url:match('[^/]-$') .. '-' .. dir_rev),
-            project_dir
-          )
-        end,
-      },
-      {
-        cmd = function()
-          vim.fn.delete(temp_dir, 'rf')
-        end,
-      },
-    }
-  else
-    local git_dir = project_dir
-    local clone_error = 'Error during download, please verify your internet connection'
-
-    return {
-      {
-        cmd = 'git',
-        info = 'Downloading ' .. project_name .. '...',
-        err = clone_error,
-        opts = {
-          args = {
-            'clone',
-            repo.url,
-            project_name,
-          },
-          cwd = cache_dir,
-        },
-      },
-      {
-        cmd = 'git',
-        info = 'Checking out locked revision',
-        err = 'Error while checking out revision',
-        opts = {
-          args = {
-            'checkout',
-            revision,
-          },
-          cwd = git_dir,
-        },
-      },
-    }
-  end
-end
-
-function M.get_package_path(...)
-  return vim.fs.joinpath(vim.fn.fnamemodify(debug.getinfo(1, 'S').source:sub(2), ':p:h:h:h'), ...)
-end
-
---TODO(clason): only needed for iter_cmd_sync -> replace with uv.spawn?
-
--- Convert path for cmd.exe on Windows (needed when shellslash is set)
----@param p string
----@return string
-local function cmdpath(p)
-  return vim.o.shellslash and p:gsub('/', '\\') or p
-end
-
----@param dir string
----@param command string
----@return string command
-function M.make_directory_change_for_command(dir, command)
-  if iswin then
-    if string.find(vim.o.shell, 'cmd') ~= nil then
-      return string.format('pushd %s & %s & popd', cmdpath(dir), command)
-    else
-      return string.format('pushd %s ; %s ; popd', cmdpath(dir), command)
-    end
-  else
-    return string.format('cd %s;\n %s', dir, command)
-  end
-end
-
-return M
diff --git a/lua/nvim-treesitter/util.lua b/lua/nvim-treesitter/util.lua
new file mode 100644
index 000000000..276eafdd7
--- /dev/null
+++ b/lua/nvim-treesitter/util.lua
@@ -0,0 +1,28 @@
+local uv = vim.loop
+
+local M = {}
+
+--- @param filename string
+--- @return string
+function M.read_file(filename)
+  local file = assert(io.open(filename, 'r'))
+  local r = file:read('*a')
+  file:close()
+  return r
+end
+
+--- @param filename string
+--- @param content string
+function M.write_file(filename, content)
+  local file = assert(io.open(filename, 'w'))
+  file:write(content)
+  file:close()
+end
+
+--- Recursively delete a directory
+--- @param name string
+function M.delete(name)
+  vim.fs.rm(name, { recursive = true, force = true })
+end
+
+return M
diff --git a/plugin/nvim-treesitter.lua b/plugin/nvim-treesitter.lua
index 3ab264d41..fcf44d537 100644
--- a/plugin/nvim-treesitter.lua
+++ b/plugin/nvim-treesitter.lua
@@ -45,19 +45,6 @@ end, {
   desc = 'Install treesitter parsers from grammar',
 })
 
-api.nvim_create_user_command('TSInstallSync', function(args)
-  require('nvim-treesitter.install').install(args.fargs, {
-    with_sync = true,
-    force = args.bang,
-  })
-end, {
-  nargs = '+',
-  bang = true,
-  bar = true,
-  complete = complete_available_parsers,
-  desc = 'Install treesitter parsers synchronously',
-})
-
 api.nvim_create_user_command('TSUpdate', function(args)
   require('nvim-treesitter.install').update(args.fargs)
 end, {
@@ -67,15 +54,6 @@ end, {
   desc = 'Update installed treesitter parsers',
 })
 
-api.nvim_create_user_command('TSUpdateSync', function(args)
-  require('nvim-treesitter.install').update(args.fargs, { with_sync = true })
-end, {
-  nargs = '*',
-  bar = true,
-  complete = complete_installed_parsers,
-  desc = 'Update installed treesitter parsers synchronously',
-})
-
 api.nvim_create_user_command('TSUninstall', function(args)
   require('nvim-treesitter.install').uninstall(args.fargs)
 end, {
@@ -84,3 +62,9 @@ end, {
   complete = complete_installed_parsers,
   desc = 'Uninstall treesitter parsers',
 })
+
+api.nvim_create_user_command('TSLog', function()
+  require('nvim-treesitter.log').show()
+end, {
+  desc = 'View log messages',
+})
diff --git a/scripts/update-readme.lua b/scripts/update-readme.lua
index 6c23c1af6..9dd9f9183 100755
--- a/scripts/update-readme.lua
+++ b/scripts/update-readme.lua
@@ -65,9 +65,8 @@ for _, v in ipairs(sorted_parsers) do
 end
 generated_text = generated_text .. footnotes
 
-local readme = assert(io.open('SUPPORTED_LANGUAGES.md', 'r'))
-local readme_text = readme:read('*a')
-readme:close()
+local readme = 'SUPPORTED_LANGUAGES.md'
+local readme_text = require('nvim-treesitter.util').read_file(readme)
 
 local new_readme_text = string.gsub(
   readme_text,
@@ -75,12 +74,10 @@ local new_readme_text = string.gsub(
   '<!--parserinfo-->\n' .. generated_text .. '<!--parserinfo-->'
 )
 
-readme = assert(io.open('SUPPORTED_LANGUAGES.md', 'w'))
-readme:write(new_readme_text)
-readme:close()
+require('nvim-treesitter.util').write_file(readme, new_readme_text)
 
 if string.find(readme_text, generated_text, 1, true) then
-  print('README.md is up-to-date\n')
+  print(readme .. ' is up-to-date\n')
 else
-  print('New README.md was written\n')
+  print('New ' .. readme .. ' was written\n')
 end
diff --git a/scripts/write-lockfile.lua b/scripts/write-lockfile.lua
index 5dac838d6..0b1d0504e 100755
--- a/scripts/write-lockfile.lua
+++ b/scripts/write-lockfile.lua
@@ -2,10 +2,8 @@
 vim.opt.runtimepath:append('.')
 
 -- Load previous lockfile
-local filename = require('nvim-treesitter.shell_cmds').get_package_path('lockfile.json')
-local file = assert(io.open(filename, 'r'))
-local lockfile = vim.json.decode(file:read('*a'))
-file:close()
+local filename = require('nvim-treesitter.install').get_package_path('lockfile.json')
+local lockfile = vim.json.decode(require('nvim-treesitter.util').read_file(filename))
 
 ---@type string?
 local skip_lang_string = os.getenv('SKIP_LOCKFILE_UPDATE_FOR_LANGS')
@@ -47,6 +45,4 @@ end
 vim.print(lockfile)
 
 -- write new lockfile
-file = assert(io.open(filename, 'w'))
-file:write(vim.json.encode(lockfile))
-file:close()
+require('nvim-treesitter.util').write_file(filename, vim.json.encode(lockfile))