cli

fossick command-line interface

---

source

fetch

def fetch(
    url:str, # URL to fetch
    sel:str=None, # CSS selector to extract (None = full page)
    heavy:bool=False, # JS rendering via headless browser
    stealthy:bool=False, # Anti-bot stealth fetcher
    session:bool=False, # Route through the persistent debug Chrome (reuses its logged-in cookies)
    auto:bool=False, # Auto-escalate plain->heavy->stealthy->session on bot-block detection
    as_json:bool=False, # Output JSON instead of markdown
):

Fetch a URL and print as markdown.

---

source

crawl

def crawl(
    url:str, # start URL
    sel:str=None, # CSS selector to extract per page
    follow_sel:str='a[href]', # CSS selector for links to follow
    max_pages:int=10, # max pages to visit
    same_domain:bool=True, # only follow same-domain links
    heavy:bool=False, # JS rendering via headless browser
    stealthy:bool=False, # anti-bot stealth fetcher
    as_json:bool=False, # output JSON
):

Crawl from a URL following links; prints each page as markdown.

---

source

search

def search(
    query:str, # search query
    n:int=10, # number of results
    region:str='us-en', # ddgs region (us-en, uk-en, ru-ru, ...)
    google:bool=False, # use direct Google via stealth browser instead of ddgs metasearch
    backend:str=None, # restrict to one ddgs backend (google, brave, duckduckgo, mojeek, yahoo, startpage)
    pages:int=1, # result pages to pull from each backend (a bigger candidate pool for large --n)
    rerank:str='lexical', # rerank the fused hits: lexical | flashrank | auto | none
    as_json:bool=False, # output JSON
):

Search the web (backends fused with RRF, or direct Google with –google); prints title, URL, and snippet.

---

source

read_arxiv

def read_arxiv(
    url:str, # arXiv URL or paper ID
    source:bool=False, # include full paper text
    chars:int=4000, # max source chars to include
    force:bool=False, # re-download even if cached
    as_json:bool=False, # output JSON
):

Fetch arXiv paper metadata, authors, and summary.

---

source

lookup_doi

def lookup_doi(
    title:str, # paper title to look up
    as_json:bool=False, # output JSON
):

Return the doi.org URL for the first Crossref match on a paper title.

---

source

read_yt

def read_yt(
    url:str, # YouTube URL or video ID
    force:bool=False, # re-fetch even if cached
    as_json:bool=False, # output JSON
):

Fetch YouTube metadata and full transcript.

---

source

search_yt

def search_yt(
    query:str, # search query
    n:int=10, # number of results
    as_json:bool=False, # output JSON
):

Search YouTube; prints title, URL, and channel.

---

source

read_gh_file

def read_gh_file(
    url:str, # GitHub blob URL of the file
    as_json:bool=False, # output JSON
):

Read the raw contents of a single GitHub file.

---

source

read_gh_repo

def read_gh_repo(
    url:str, # GitHub repo URL, SSH address, or local path
    globs:str=None, # comma-sep glob patterns (default: README*,pyproject.toml,*.py)
    limit:int=None, # max files to return
    as_json:bool=False, # output JSON
):

Read files from a GitHub repo filtered by glob patterns.

---

source

calls

def calls(
    url:str, # URL to navigate to
    pattern:str='.*', # regex/glob to filter request URLs
    tail:int=3, # seconds to wait after navigation
    as_json:bool=False, # output JSON
):

Capture outgoing network requests fired by a page.

---

source

find_xhr

def find_xhr(
    url:str, # URL to visit with a headless browser
    pattern:str='*', # glob/regex to filter captured XHR URLs
    as_json:bool=False, # output JSON
):

Discover hidden JSON/XHR API calls a page makes.

---

source

paginate_api

def paginate_api(
    url:str, # API endpoint URL
    payload:str=None, # request body/params as a JSON string
    page_field:str='pageNumber', # payload key to increment per page
    results_field:str=None, # response key with the items list (auto-detect if None)
    method:str='POST', # HTTP method
    max_pages:int=10, # max pages to fetch
):

Paginate a JSON API and print all collected items as JSON.

---

source

collect

def collect(
    url:str, # URL to open in Chrome
    save_dir:str='.', # directory to save screenshots
    tout:int=None, # stop after this many seconds
    count:int=None, # stop after this many screenshots
    every_n:int=None, # auto-capture every N seconds
):

Open a URL in Chrome and capture screenshots interactively.

---

source

annotate

def annotate(
    url:str, # URL to open in Chrome
    save_dir:str='.', # directory to save annotated screenshot
    as_json:bool=False, # output JSON list of selected elements
):

Open a URL in Chrome, click elements to annotate them, save labeled screenshot.

---

source

install

def install():

Install fossick SKILL.md to .agents/skills/fossick/ and .claude/skills/fossick/.

---

source

url2nb

def url2nb(
    url:str, # URL to convert (HTML page, PDF, or arXiv)
    path:str=None, # output notebook path (default: derived from URL)
    as_json:bool=False, # output JSON with notebook path
):

Convert a URL (HTML, PDF, or arXiv) to a Jupyter notebook.

---

source

pdf2nb

def pdf2nb(
    url:str, # PDF URL or local path
    path:str=None, # output notebook path (default: derived from PDF)
    ocr:str='auto', # OCR mode: auto | on | off
    force:bool=False, # re-convert even if notebook exists
    as_json:bool=False, # output JSON with notebook path
):

Convert a PDF (URL or local path) to a Jupyter notebook.

---

source

download_yt

def download_yt(
    url:str, # YouTube URL or video ID
    format:str='audio', # audio | video | yt-dlp format string
    save_dir:str='.', # directory to save to
    as_json:bool=False, # output JSON with saved path
):

Download YouTube audio or video.

---

source

videos

def videos(
    query:str, n:int=20, region:str='us-en', as_json:bool=False
):

Video search via ddgs; prints title and video URL.

---

source

news

def news(
    query:str, n:int=20, region:str='us-en', as_json:bool=False
):

News search via ddgs; prints date, title, and URL.

---

source

images

def images(
    query:str, n:int=20, region:str='us-en', as_json:bool=False
):

Image search via ddgs; prints title and image URL.

---

source

ax

def ax(
    url:str, # URL to open in the debug Chrome
    port:int=9223, # debug Chrome remote-debugging port
    full:bool=False, # show the full accessibility tree instead of interactive-only
):

Open a URL in the persistent debug Chrome and print a compact, agent-ready accessibility snapshot.

---

source

research

def research(
    query:str, # search query
    n:int=5, # number of top results to read
    google:bool=False, # use direct Google ranking (stealth browser) instead of ddgs metasearch
    region:str='auto', # ddgs 'country-lang' region, or 'auto' to read the country off the query
    timelimit:str=None, # d | w | m | y — only sources from the last day/week/month/year
    sel:str=None, # CSS selector to narrow each page before markdown
    chars:int=4000, # max markdown chars per source
    pages:int=1, # result pages to pull from each backend when searching
    as_json:bool=False, # output JSON
):

Search, then read the top results into one cited markdown corpus.

---

source

shop

def shop(
    url:str, # store URL; the tab the last call left open on that site is picked up again
    search:str=None, # search the store for this term and list the products it found
    add:str=None, # add a product by index (from a previous --search) or by title
    qty:int=1, # quantity to add
    variant:str=None, # variant/size to pick, for products that need one
    cart:bool=False, # print the cart (with its lines) instead
    fields:bool=False, # print the form fields on this page (checkout ground truth)
    port:int=9223, # debug Chrome remote-debugging port
    as_json:bool=False, # output JSON
):

Drive a shopping cart: list products, add one, read the cart. Page and cart state persist across calls.

---

source

main

def main():

Entry point for the fossick CLI command.

Tests

# end-to-end: the research CLI runs against live services and prints a cited corpus
import io
from contextlib import redirect_stdout
buf = io.StringIO()
with redirect_stdout(buf): research.__wrapped__('what is the raft consensus algorithm', n=2)
out = buf.getvalue()
assert out.startswith('# Research:') and 'http' in out
print(out[:300])