feat(tools): add fetch_webpage (#1837)

olimorris · web-flow · commit 31fdc343e8ab · 2025-07-22T10:06:02.000+01:00
diff --git a/doc/usage/chat-buffer/agents.md b/doc/usage/chat-buffer/agents.md
@@ -74,6 +74,17 @@ Can you create some test fixtures using the @{create_file} tool?
 **Options:**
 - `requires_approval` require approval before creating a file? (Default: true)
 
+## fetch_webpage
+
+This tools enables an LLM to fetch the content from a specific webpage. It will return the text in a text format, depending on which adapter you've configured for the tool.
+
+```md
+Use the @{fetch_webpage} tool to tell me what the latest version on neovim.io is
+```
+
+**Options:**
+- `adapter` The adapter used to fetch, process and format the webpage's content (Default: `jina`)
+
 ## file_search
 
 This tool enables an LLM to search for files in the current working directory by glob pattern. It will return a list of relative paths for any matching files.
diff --git a/lua/codecompanion/adapters/jina.lua b/lua/codecompanion/adapters/jina.lua
@@ -1,10 +1,13 @@
+local log = require("codecompanion.utils.log")
+
 ---@class CodeCompanion.AdapterArgs
 return {
   name = "jina",
   opts = {
     stream = false,
   },
   url = "",
+  env = {},
   headers = {},
   schema = {
     model = {
@@ -37,5 +40,44 @@ return {
         end
       end,
     },
+    tools = {
+      fetch_webpage = {
+        ---Setup the adapter for the fetch webpage tool
+        ---@param self CodeCompanion.Adapter
+        ---@param data table The data from the LLM's tool call
+        ---@return nil
+        setup = function(self, data)
+          self.methods.slash_commands.fetch(self, data)
+          self.handlers.set_body = function()
+            return { url = data.url }
+          end
+        end,
+
+        ---Process the output from the fetch webpage tool
+        ---@param self CodeCompanion.Adapter
+        ---@param data table The data returned from the fetch
+        ---@return table{status: string, content: string}|nil
+        callback = function(self, data)
+          local ok, data = pcall(vim.json.decode, data.body)
+          if not ok then
+            return {
+              status = "error",
+              output = "Failed to decode JSON content",
+            }
+          end
+          if data.code ~= 200 then
+            log:error("[Jina Adapter] Error: %s", data)
+            return {
+              status = "error",
+              content = data.message or data.body or "Unknown error occurred",
+            }
+          end
+          return {
+            success = "success",
+            content = data.data.text,
+          }
+        end,
+      },
+    },
   },
 }
diff --git a/lua/codecompanion/config.lua b/lua/codecompanion/config.lua
@@ -99,6 +99,13 @@ local defaults = {
             requires_approval = true,
           },
         },
+        ["fetch_webpage"] = {
+          callback = "strategies.chat.agents.tools.fetch_webpage",
+          description = "Fetches content from a webpage",
+          opts = {
+            adapter = "jina",
+          },
+        },
         ["file_search"] = {
           callback = "strategies.chat.agents.tools.file_search",
           description = "Search for files in the current working directory by glob pattern",
@@ -167,6 +174,7 @@ local defaults = {
               "cancelled",
               "error",
               "failed",
+              "incorrect",
               "invalid",
               "rejected",
             },
diff --git a/lua/codecompanion/strategies/chat/agents/tools/fetch_webpage.lua b/lua/codecompanion/strategies/chat/agents/tools/fetch_webpage.lua
@@ -0,0 +1,123 @@
+local adapters = require("codecompanion.adapters")
+local client = require("codecompanion.http")
+local config = require("codecompanion.config")
+local log = require("codecompanion.utils.log")
+
+local fmt = string.format
+
+---@class CodeCompanion.Tool.FetchWebpage: CodeCompanion.Agent.Tool
+return {
+  name = "fetch_webpage",
+  cmds = {
+    ---Execute the fetch_webpage tool
+    ---@param self CodeCompanion.Agent
+    ---@param args table The arguments from the LLM's tool call
+    ---@param cb function Async callback for completion
+    ---@return nil
+    function(self, args, _, cb)
+      local opts = self.tool.opts
+      local url = args.url
+
+      if not opts or not opts.adapter then
+        log:error("[Fetch Webpage Tool] No adapter set for `fetch_webpage`")
+        return cb({ status = "error", data = "No adapter for `fetch_webpage`" })
+      end
+      if not args then
+        log:error("[Fetch Webpage Tool] No args for `fetch_webpage`")
+        return cb({ status = "error", data = "No args for `fetch_webpage`" })
+      end
+
+      if not url or type(url) ~= "string" or url == "" then
+        return cb({ status = "error", data = fmt("No URL for `fetch_webpage`") })
+      end
+
+      local tool_adapter = config.strategies.chat.tools.fetch_webpage.opts.adapter
+      local adapter = vim.deepcopy(adapters.resolve(config.adapters[tool_adapter]))
+      adapter.methods.tools.fetch_webpage.setup(adapter, args)
+
+      if not url:match("^https?://") then
+        log:error("[Fetch Webpage Tool] Invalid URL: `%s`", url)
+        return cb({ status = "error", data = fmt("Invalid URL: `%s`", url) })
+      end
+
+      client
+        .new({
+          adapter = adapter,
+        })
+        :request(_, {
+          callback = function(err, data)
+            if err then
+              log:error("[Fetch Webpage Tool] Error fetching `%s`: %s", url, err)
+              return cb({ status = "error", data = fmt("Error fetching `%s`\n%s", url, err) })
+            end
+
+            if data then
+              local output = adapter.methods.tools.fetch_webpage.callback(adapter, data)
+              if output.status == "error" then
+                log:error("[Fetch Webpage Tool] Error processing data for `%s`: %s", url, output.content)
+                return cb({ status = "error", data = fmt("Error processing `%s`\n%s", url, output.content) })
+              end
+
+              return cb({ status = "success", data = output.content })
+            end
+          end,
+        })
+    end,
+  },
+  schema = {
+    type = "function",
+    ["function"] = {
+      name = "fetch_webpage",
+      description = "Fetches the main content from a web page. You should use this tool when you think the user is looking for information from a specific webpage.",
+      parameters = {
+        type = "object",
+        properties = {
+          url = {
+            type = "string",
+            description = "The URL of the webpage to fetch content from",
+          },
+        },
+        required = { "url" },
+      },
+    },
+  },
+  output = {
+    ---@param self CodeCompanion.Tool.FetchWebpage
+    ---@param agent CodeCompanion.Agent
+    ---@param cmd table The command that was executed
+    ---@param stdout table The output from the command
+    success = function(self, agent, cmd, stdout)
+      local args = self.args
+      local chat = agent.chat
+
+      local content = vim.iter(stdout):flatten():join("\n")
+
+      local llm_output = fmt([[<fetchWebpageTool url="%s">%s</fetchWebpageTool>]], args.url, content)
+      local user_output = fmt("Fetched content from `%s`", args.url)
+
+      chat:add_tool_output(self, llm_output, user_output)
+    end,
+
+    ---@param self CodeCompanion.Tool.FetchWebpage
+    ---@param agent CodeCompanion.Agent
+    ---@param cmd table
+    ---@param stderr table The error output from the command
+    ---@param stdout? table The output from the command
+    error = function(self, agent, cmd, stderr, stdout)
+      local args = self.args
+      local chat = agent.chat
+      local errors = vim.iter(stderr):flatten():join("\n")
+      log:debug("[Fetch Webpage Tool] Error output: %s", stderr)
+
+      local error_output = fmt(
+        [[Error fetching content from `%s`:
+```txt
+%s
+```]],
+        args.url,
+        errors
+      )
+      chat:add_tool_output(self, error_output)
+    end,
+  },
+}
diff --git a/tests/config.lua b/tests/config.lua
@@ -89,6 +89,13 @@ return {
           callback = "strategies.chat.agents.tools.create_file",
           description = "Create a file in the current working directory",
         },
+        ["fetch_webpage"] = {
+          callback = "strategies.chat.agents.tools.fetch_webpage",
+          description = "Fetches content from a webpage",
+          opts = {
+            adapter = "jina",
+          },
+        },
         ["file_search"] = {
           callback = "strategies.chat.agents.tools.file_search",
           description = "Search for files in the current working directory by glob pattern",
diff --git a/tests/strategies/chat/agents/tools/test_fetch_webpage.lua b/tests/strategies/chat/agents/tools/test_fetch_webpage.lua
@@ -0,0 +1,97 @@
+local h = require("tests.helpers")
+local new_set = MiniTest.new_set
+
+local child = MiniTest.new_child_neovim()
+local T = new_set({
+  hooks = {
+    pre_case = function()
+      h.child_start(child)
+      child.lua([[
+        h = require('tests.helpers')
+        chat, agent = h.setup_chat_buffer()
+
+        -- Store originals for restoration
+        _G.original_http = package.loaded["codecompanion.http"]
+        _G.original_adapters = package.loaded["codecompanion.adapters"]
+
+        -- Minimal mock adapter - just enough to not crash
+        local mock_adapter = {
+          methods = {
+            tools = {
+              fetch_webpage = {
+                setup = function() end,
+                callback = function(adapter, data)
+                  -- Let the real tool handle the formatting
+                  return { status = "success", content = data.html or "mock content" }
+                end
+              }
+            }
+          }
+        }
+
+        package.loaded["codecompanion.adapters"] = {
+          resolve = function() return mock_adapter end
+        }
+
+        -- Minimal HTTP mock - just return some data
+        package.loaded["codecompanion.http"] = {
+          new = function()
+            return {
+              request = function(_, __, handlers)
+                vim.schedule(function()
+                  handlers.callback(nil, { html = "Hello World content" })
+                end)
+              end
+            }
+          end
+        }
+
+        -- Minimal config
+        local config = require("codecompanion.config")
+        config.strategies.chat.tools.fetch_webpage = {
+          callback = "strategies.chat.agents.tools.fetch_webpage",
+          opts = { adapter = "test_adapter" }
+        }
+        config.adapters.test_adapter = {}
+      ]])
+    end,
+    post_case = function()
+      child.lua([[
+        -- Restore original modules
+        package.loaded["codecompanion.http"] = _G.original_http
+        package.loaded["codecompanion.adapters"] = _G.original_adapters
+        h.teardown_chat_buffer()
+      ]])
+    end,
+    post_once = child.stop,
+  },
+})
+
+T["fetches webpage content successfully"] = function()
+  child.lua([[
+    local tool_call = {
+      ["function"] = {
+        name = "fetch_webpage",
+        arguments = '{"url": "https://example.com"}'
+      },
+      id = "test_call_id",
+      type = "function"
+    }
+
+    -- Execute the real agent with real tool
+    agent:execute(chat, { tool_call })
+    vim.wait(500) -- Give time for async operations
+  ]])
+
+  local messages = child.lua_get("chat.messages")
+  local tool_message = vim.tbl_filter(function(msg)
+    return msg.role == "tool"
+  end, messages)[1]
+
+  -- Match the actual output format
+  h.expect_contains('<fetchWebpageTool url="https://example.com">', tool_message.content)
+  h.expect_contains("Hello World content", tool_message.content)
+  h.expect_contains("</fetchWebpageTool>", tool_message.content)
+end
+
+return T
