Skip to content

Commit 9254e30

Browse files
vdusekvdusek
committed
chore: Migrate to ty type checker
1 parent 6833540 commit 9254e30

File tree

59 files changed

+268
-369
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

59 files changed

+268
-369
lines changed

‎.gitignore‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# Cache
22
__pycache__
3-
.mypy_cache
3+
.uv_cache
44
.pytest_cache
55
.ruff_cache
66
.uv-cache

‎CONTRIBUTING.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ make format
4646

4747
### Type checking
4848

49-
Type checking is handled by [mypy](https://mypy.readthedocs.io/), verifying code against type annotations. Configuration settings can be found in `pyproject.toml`.
49+
Type checking is handled by [ty](https://docs.astral.sh/ty/), verifying code against type annotations. Configuration settings can be found in `pyproject.toml`.
5050

5151
To run type checking:
5252

‎Makefile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
E2E_TESTS_CONCURRENCY = 1
66

77
clean:
8-
rm -rf .mypy_cache .pytest_cache .ruff_cache .uv-cache build dist htmlcov .coverage
8+
rm -rf .uv_cache .pytest_cache .ruff_cache .uv-cache build dist htmlcov .coverage
99

1010
install-sync:
1111
uv sync --all-extras
@@ -27,7 +27,7 @@ lint:
2727
uv run ruff check
2828

2929
type-check:
30-
uv run mypy
30+
uv run ty check
3131

3232
unit-tests:
3333
uv run pytest \

‎docs/deployment/code_examples/google/cloud_run_example.py‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
# mypy: disable-error-code="misc"
21
import json
32
import os
43

@@ -9,7 +8,7 @@
98
from crawlee.storage_clients import MemoryStorageClient
109

1110

12-
@get('/') # type: ignore[untyped-decorator]
11+
@get('/')
1312
async def main() -> str:
1413
"""The crawler entry point that will be called when the HTTP endpoint is accessed."""
1514
# highlight-start

‎docs/deployment/code_examples/google/google_example.py‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
# mypy: disable-error-code="misc"
21
import asyncio
32
import json
43
from datetime import timedelta
@@ -48,7 +47,7 @@ async def request_handler(context: BeautifulSoupCrawlingContext) -> None:
4847
# highlight-end
4948

5049

51-
@functions_framework.http # type: ignore[untyped-decorator]
50+
@functions_framework.http
5251
def crawlee_run(request: Request) -> Response:
5352
# You can pass data to your crawler using `request`
5453
function_id = request.headers['Function-Execution-Id']

‎docs/guides/code_examples/avoid_blocking/default_fingerprint_generator_with_args.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99

1010
async def main() -> None:
1111
fingerprint_generator = DefaultFingerprintGenerator(
12-
header_options=HeaderGeneratorOptions(browsers=['chromium']),
12+
header_options=HeaderGeneratorOptions(browsers=['chrome']),
1313
screen_options=ScreenOptions(min_width=400),
1414
)
1515

‎docs/guides/code_examples/running_in_web_server/server.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
app = FastAPI(lifespan=lifespan, title='Crawler app')
1515

1616

17-
@app.get('/', response_class=HTMLResponse) # type: ignore[untyped-decorator]
17+
@app.get('/', response_class=HTMLResponse)
1818
def index() -> str:
1919
return """
2020
<!DOCTYPE html>
@@ -32,7 +32,7 @@ def index() -> str:
3232
"""
3333

3434

35-
@app.get('/scrape') # type: ignore[untyped-decorator]
35+
@app.get('/scrape')
3636
async def scrape_url(request: Request, url: str | None = None) -> dict:
3737
if not url:
3838
return {'url': 'missing', 'scrape result': 'no results'}

‎pyproject.toml‎

Lines changed: 12 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,6 @@ dev = [
102102
"build<2.0.0", # For e2e tests.
103103
"dycw-pytest-only<3.0.0",
104104
"fakeredis[probabilistic,json,lua]<3.0.0",
105-
"mypy~=1.19.0",
106105
"pre-commit<5.0.0",
107106
"proxy-py<3.0.0",
108107
"pydoc-markdown<5.0.0",
@@ -113,6 +112,7 @@ dev = [
113112
"pytest<9.0.0",
114113
"ruff~=0.14.0",
115114
"setuptools", # setuptools are used by pytest, but not explicitly required
115+
"ty~=0.0.0",
116116
"types-beautifulsoup4<5.0.0",
117117
"types-cachetools<7.0.0",
118118
"types-colorama<1.0.0",
@@ -230,57 +230,24 @@ filterwarnings = [
230230
"ignore:websockets.server.WebSocketServerProtocol is deprecated:DeprecationWarning",
231231
]
232232

233-
[tool.mypy]
234-
python_version = "3.10"
235-
plugins = ["pydantic.mypy"]
233+
[tool.ty.environment]
234+
python-version = "3.10"
235+
236+
[tool.ty.src]
237+
include = ["src", "tests", "scripts", "docs", "website"]
236238
exclude = [
237239
"src/crawlee/project_template",
238240
"docs/guides/code_examples/storage_clients/custom_storage_client_example.py",
239241
]
240-
files = ["src", "tests", "docs", "website"]
241-
check_untyped_defs = true
242-
disallow_incomplete_defs = true
243-
disallow_untyped_calls = true
244-
disallow_untyped_decorators = true
245-
disallow_untyped_defs = true
246-
no_implicit_optional = true
247-
warn_redundant_casts = true
248-
warn_return_any = true
249-
warn_unreachable = true
250-
warn_unused_ignores = true
251-
252-
[[tool.mypy.overrides]]
253-
# Example codes are sometimes showing integration of crawlee with external tool, which is not dependency of crawlee.
254-
module = [
255-
"apify", # Example code shows integration of apify and crawlee.
256-
"apify_fingerprint_datapoints", # Untyped and stubs not available
257-
"camoufox", # Example code shows integration of camoufox and crawlee.
258-
"fastapi", # Example code shows running in webserver.
259-
"stagehand.*", # Example code shows integration of Stagehand and crawlee.
260-
"starlette.*", # Example code shows running in webserver.
261-
"flask", # Example code shows deploy on Google Cloud.
262-
"functions_framework", # Example code shows deploy on Google Cloud.
263-
"jaro", # Untyped and stubs not available
264-
"litestar", # Example code shows deploy on Google Cloud Run.
265-
"loguru", # Example code shows integration of loguru and crawlee for JSON logging.
266-
"sklearn.linear_model", # Untyped and stubs not available
267-
"cookiecutter.*", # Untyped and stubs not available
268-
"inquirer.*", # Untyped and stubs not available
269-
"warcio.*", # Example code shows WARC files creation.
270-
"wrapt" # Untyped and stubs not available
271-
]
272-
ignore_missing_imports = true
273242

274-
[[tool.mypy.overrides]]
275-
module = [
276-
"running_in_web_server.*", # False positive when fastapi not available
243+
[[tool.ty.overrides]]
244+
include = [
245+
"docs/**/*.py",
246+
"website/**/*.py",
277247
]
278-
disable_error_code = ["misc"]
279248

280-
[tool.basedpyright]
281-
pythonVersion = "3.10"
282-
typeCheckingMode = "standard"
283-
include = ["src", "tests", "docs", "website"]
249+
[tool.ty.overrides.rules]
250+
unresolved-import = "ignore"
284251

285252
[tool.coverage.report]
286253
exclude_lines = ["pragma: no cover", "if TYPE_CHECKING:", "assert_never()"]

‎src/crawlee/_browserforge_workaround.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ def patch_browserforge() -> None:
2020
def DownloadIfNotExists(**flags: bool) -> None:
2121
pass
2222

23-
download.DownloadIfNotExists = DownloadIfNotExists
23+
download.DownloadIfNotExists = DownloadIfNotExists # ty: ignore[invalid-assignment]
2424

2525
import browserforge.bayesian_network
2626

‎src/crawlee/_request.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@ def __setitem__(self, key: str, value: JsonSerializable) -> None:
9393
def __delitem__(self, key: str) -> None:
9494
del self.__pydantic_extra__[key]
9595

96-
def __iter__(self) -> Iterator[str]: # type: ignore[override]
96+
def __iter__(self) -> Iterator[str]: # ty: ignore[invalid-method-override]
9797
yield from self.__pydantic_extra__
9898

9999
def __len__(self) -> int:
@@ -195,7 +195,7 @@ class Request(BaseModel):
195195
] = None
196196
"""HTTP request payload."""
197197

198-
# Workaround for pydantic 2.12 and mypy type checking issue for Annotated with default_factory
198+
# Workaround for Pydantic and type checkers when using Annotated with default_factory
199199
if TYPE_CHECKING:
200200
headers: HttpHeaders = HttpHeaders()
201201
"""HTTP request headers."""

0 commit comments

Comments
 (0)