Initial commit
This commit is contained in:
commit
c96063470a
14 changed files with 830 additions and 0 deletions
1
.envrc
Normal file
1
.envrc
Normal file
|
@ -0,0 +1 @@
|
||||||
|
eval "$(lorri direnv)"
|
147
.gitignore
vendored
Normal file
147
.gitignore
vendored
Normal file
|
@ -0,0 +1,147 @@
|
||||||
|
|
||||||
|
# Created by https://www.toptal.com/developers/gitignore/api/python
|
||||||
|
# Edit at https://www.toptal.com/developers/gitignore?templates=python
|
||||||
|
|
||||||
|
### Python ###
|
||||||
|
# Byte-compiled / optimized / DLL files
|
||||||
|
__pycache__/
|
||||||
|
*.py[cod]
|
||||||
|
*$py.class
|
||||||
|
|
||||||
|
# C extensions
|
||||||
|
*.so
|
||||||
|
|
||||||
|
# Distribution / packaging
|
||||||
|
.Python
|
||||||
|
build/
|
||||||
|
develop-eggs/
|
||||||
|
dist/
|
||||||
|
downloads/
|
||||||
|
eggs/
|
||||||
|
.eggs/
|
||||||
|
lib/
|
||||||
|
lib64/
|
||||||
|
parts/
|
||||||
|
sdist/
|
||||||
|
var/
|
||||||
|
wheels/
|
||||||
|
share/python-wheels/
|
||||||
|
*.egg-info/
|
||||||
|
.installed.cfg
|
||||||
|
*.egg
|
||||||
|
MANIFEST
|
||||||
|
|
||||||
|
# PyInstaller
|
||||||
|
# Usually these files are written by a python script from a template
|
||||||
|
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
||||||
|
*.manifest
|
||||||
|
*.spec
|
||||||
|
|
||||||
|
# Installer logs
|
||||||
|
pip-log.txt
|
||||||
|
pip-delete-this-directory.txt
|
||||||
|
|
||||||
|
# Unit test / coverage reports
|
||||||
|
htmlcov/
|
||||||
|
.tox/
|
||||||
|
.nox/
|
||||||
|
.coverage
|
||||||
|
.coverage.*
|
||||||
|
.cache
|
||||||
|
nosetests.xml
|
||||||
|
coverage.xml
|
||||||
|
*.cover
|
||||||
|
*.py,cover
|
||||||
|
.hypothesis/
|
||||||
|
.pytest_cache/
|
||||||
|
cover/
|
||||||
|
|
||||||
|
# Translations
|
||||||
|
*.mo
|
||||||
|
*.pot
|
||||||
|
|
||||||
|
# Django stuff:
|
||||||
|
*.log
|
||||||
|
local_settings.py
|
||||||
|
db.sqlite3
|
||||||
|
db.sqlite3-journal
|
||||||
|
|
||||||
|
# Flask stuff:
|
||||||
|
instance/
|
||||||
|
.webassets-cache
|
||||||
|
|
||||||
|
# Scrapy stuff:
|
||||||
|
.scrapy
|
||||||
|
|
||||||
|
# Sphinx documentation
|
||||||
|
docs/_build/
|
||||||
|
|
||||||
|
# PyBuilder
|
||||||
|
.pybuilder/
|
||||||
|
target/
|
||||||
|
|
||||||
|
# Jupyter Notebook
|
||||||
|
.ipynb_checkpoints
|
||||||
|
|
||||||
|
# IPython
|
||||||
|
profile_default/
|
||||||
|
ipython_config.py
|
||||||
|
|
||||||
|
# pyenv
|
||||||
|
# For a library or package, you might want to ignore these files since the code is
|
||||||
|
# intended to run in multiple environments; otherwise, check them in:
|
||||||
|
# .python-version
|
||||||
|
|
||||||
|
# pipenv
|
||||||
|
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
||||||
|
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
||||||
|
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
||||||
|
# install all needed dependencies.
|
||||||
|
#Pipfile.lock
|
||||||
|
|
||||||
|
# PEP 582; used by e.g. github.com/David-OConnor/pyflow
|
||||||
|
__pypackages__/
|
||||||
|
|
||||||
|
# Celery stuff
|
||||||
|
celerybeat-schedule
|
||||||
|
celerybeat.pid
|
||||||
|
|
||||||
|
# SageMath parsed files
|
||||||
|
*.sage.py
|
||||||
|
|
||||||
|
# Environments
|
||||||
|
.env
|
||||||
|
.venv
|
||||||
|
env/
|
||||||
|
venv/
|
||||||
|
ENV/
|
||||||
|
env.bak/
|
||||||
|
venv.bak/
|
||||||
|
|
||||||
|
# Spyder project settings
|
||||||
|
.spyderproject
|
||||||
|
.spyproject
|
||||||
|
|
||||||
|
# Rope project settings
|
||||||
|
.ropeproject
|
||||||
|
|
||||||
|
# mkdocs documentation
|
||||||
|
/site
|
||||||
|
|
||||||
|
# mypy
|
||||||
|
.mypy_cache/
|
||||||
|
.dmypy.json
|
||||||
|
dmypy.json
|
||||||
|
|
||||||
|
# Pyre type checker
|
||||||
|
.pyre/
|
||||||
|
|
||||||
|
# pytype static type analyzer
|
||||||
|
.pytype/
|
||||||
|
|
||||||
|
# Cython debug symbols
|
||||||
|
cython_debug/
|
||||||
|
|
||||||
|
# End of https://www.toptal.com/developers/gitignore/api/python
|
||||||
|
|
||||||
|
result/
|
50
nix/sources.json
Normal file
50
nix/sources.json
Normal file
|
@ -0,0 +1,50 @@
|
||||||
|
{
|
||||||
|
"libnixstore-c": {
|
||||||
|
"branch": "master",
|
||||||
|
"description": "a brutal attempt at making C bindings for Nix",
|
||||||
|
"homepage": null,
|
||||||
|
"owner": "andir",
|
||||||
|
"repo": "libnixstore-c",
|
||||||
|
"rev": "7e680c54ad0bdc38a4f6a6a51e4d6427d8942bb3",
|
||||||
|
"sha256": "1a8nxcpp93hly410ni77q1zw5097dpvkszw3h6l5cr25wnb31ngq",
|
||||||
|
"type": "tarball",
|
||||||
|
"url": "https://github.com/andir/libnixstore-c/archive/7e680c54ad0bdc38a4f6a6a51e4d6427d8942bb3.tar.gz",
|
||||||
|
"url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
|
||||||
|
},
|
||||||
|
"mach-nix": {
|
||||||
|
"branch": "master",
|
||||||
|
"description": "Create highly reproducible python environments",
|
||||||
|
"homepage": "",
|
||||||
|
"owner": "DavHau",
|
||||||
|
"repo": "mach-nix",
|
||||||
|
"rev": "b56a541af15efd2062ffb9abb69f63dcceafb64d",
|
||||||
|
"sha256": "0zdifqdq478q938wm3pwdph8xv9ksk9qvf6s3kckchyzw18x28k0",
|
||||||
|
"type": "tarball",
|
||||||
|
"url": "https://github.com/DavHau/mach-nix/archive/b56a541af15efd2062ffb9abb69f63dcceafb64d.tar.gz",
|
||||||
|
"url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
|
||||||
|
},
|
||||||
|
"niv": {
|
||||||
|
"branch": "master",
|
||||||
|
"description": "Easy dependency management for Nix projects",
|
||||||
|
"homepage": "https://github.com/nmattia/niv",
|
||||||
|
"owner": "nmattia",
|
||||||
|
"repo": "niv",
|
||||||
|
"rev": "65a61b147f307d24bfd0a5cd56ce7d7b7cc61d2e",
|
||||||
|
"sha256": "17mirpsx5wyw262fpsd6n6m47jcgw8k2bwcp1iwdnrlzy4dhcgqh",
|
||||||
|
"type": "tarball",
|
||||||
|
"url": "https://github.com/nmattia/niv/archive/65a61b147f307d24bfd0a5cd56ce7d7b7cc61d2e.tar.gz",
|
||||||
|
"url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
|
||||||
|
},
|
||||||
|
"nixpkgs": {
|
||||||
|
"branch": "nixos-unstable",
|
||||||
|
"description": "Nix Packages collection",
|
||||||
|
"homepage": "",
|
||||||
|
"owner": "NixOS",
|
||||||
|
"repo": "nixpkgs",
|
||||||
|
"rev": "2cf9db0e3d45b9d00f16f2836cb1297bcadc475e",
|
||||||
|
"sha256": "0sij1a5hlbigwcgx10dkw6mdbjva40wzz4scn0wchv7yyi9ph48l",
|
||||||
|
"type": "tarball",
|
||||||
|
"url": "https://github.com/NixOS/nixpkgs/archive/2cf9db0e3d45b9d00f16f2836cb1297bcadc475e.tar.gz",
|
||||||
|
"url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
|
||||||
|
}
|
||||||
|
}
|
174
nix/sources.nix
Normal file
174
nix/sources.nix
Normal file
|
@ -0,0 +1,174 @@
|
||||||
|
# This file has been generated by Niv.
|
||||||
|
|
||||||
|
let
|
||||||
|
|
||||||
|
#
|
||||||
|
# The fetchers. fetch_<type> fetches specs of type <type>.
|
||||||
|
#
|
||||||
|
|
||||||
|
fetch_file = pkgs: name: spec:
|
||||||
|
let
|
||||||
|
name' = sanitizeName name + "-src";
|
||||||
|
in
|
||||||
|
if spec.builtin or true then
|
||||||
|
builtins_fetchurl { inherit (spec) url sha256; name = name'; }
|
||||||
|
else
|
||||||
|
pkgs.fetchurl { inherit (spec) url sha256; name = name'; };
|
||||||
|
|
||||||
|
fetch_tarball = pkgs: name: spec:
|
||||||
|
let
|
||||||
|
name' = sanitizeName name + "-src";
|
||||||
|
in
|
||||||
|
if spec.builtin or true then
|
||||||
|
builtins_fetchTarball { name = name'; inherit (spec) url sha256; }
|
||||||
|
else
|
||||||
|
pkgs.fetchzip { name = name'; inherit (spec) url sha256; };
|
||||||
|
|
||||||
|
fetch_git = name: spec:
|
||||||
|
let
|
||||||
|
ref =
|
||||||
|
if spec ? ref then spec.ref else
|
||||||
|
if spec ? branch then "refs/heads/${spec.branch}" else
|
||||||
|
if spec ? tag then "refs/tags/${spec.tag}" else
|
||||||
|
abort "In git source '${name}': Please specify `ref`, `tag` or `branch`!";
|
||||||
|
in
|
||||||
|
builtins.fetchGit { url = spec.repo; inherit (spec) rev; inherit ref; };
|
||||||
|
|
||||||
|
fetch_local = spec: spec.path;
|
||||||
|
|
||||||
|
fetch_builtin-tarball = name: throw
|
||||||
|
''[${name}] The niv type "builtin-tarball" is deprecated. You should instead use `builtin = true`.
|
||||||
|
$ niv modify ${name} -a type=tarball -a builtin=true'';
|
||||||
|
|
||||||
|
fetch_builtin-url = name: throw
|
||||||
|
''[${name}] The niv type "builtin-url" will soon be deprecated. You should instead use `builtin = true`.
|
||||||
|
$ niv modify ${name} -a type=file -a builtin=true'';
|
||||||
|
|
||||||
|
#
|
||||||
|
# Various helpers
|
||||||
|
#
|
||||||
|
|
||||||
|
# https://github.com/NixOS/nixpkgs/pull/83241/files#diff-c6f540a4f3bfa4b0e8b6bafd4cd54e8bR695
|
||||||
|
sanitizeName = name:
|
||||||
|
(
|
||||||
|
concatMapStrings (s: if builtins.isList s then "-" else s)
|
||||||
|
(
|
||||||
|
builtins.split "[^[:alnum:]+._?=-]+"
|
||||||
|
((x: builtins.elemAt (builtins.match "\\.*(.*)" x) 0) name)
|
||||||
|
)
|
||||||
|
);
|
||||||
|
|
||||||
|
# The set of packages used when specs are fetched using non-builtins.
|
||||||
|
mkPkgs = sources: system:
|
||||||
|
let
|
||||||
|
sourcesNixpkgs =
|
||||||
|
import (builtins_fetchTarball { inherit (sources.nixpkgs) url sha256; }) { inherit system; };
|
||||||
|
hasNixpkgsPath = builtins.any (x: x.prefix == "nixpkgs") builtins.nixPath;
|
||||||
|
hasThisAsNixpkgsPath = <nixpkgs> == ./.;
|
||||||
|
in
|
||||||
|
if builtins.hasAttr "nixpkgs" sources
|
||||||
|
then sourcesNixpkgs
|
||||||
|
else if hasNixpkgsPath && ! hasThisAsNixpkgsPath then
|
||||||
|
import <nixpkgs> {}
|
||||||
|
else
|
||||||
|
abort
|
||||||
|
''
|
||||||
|
Please specify either <nixpkgs> (through -I or NIX_PATH=nixpkgs=...) or
|
||||||
|
add a package called "nixpkgs" to your sources.json.
|
||||||
|
'';
|
||||||
|
|
||||||
|
# The actual fetching function.
|
||||||
|
fetch = pkgs: name: spec:
|
||||||
|
|
||||||
|
if ! builtins.hasAttr "type" spec then
|
||||||
|
abort "ERROR: niv spec ${name} does not have a 'type' attribute"
|
||||||
|
else if spec.type == "file" then fetch_file pkgs name spec
|
||||||
|
else if spec.type == "tarball" then fetch_tarball pkgs name spec
|
||||||
|
else if spec.type == "git" then fetch_git name spec
|
||||||
|
else if spec.type == "local" then fetch_local spec
|
||||||
|
else if spec.type == "builtin-tarball" then fetch_builtin-tarball name
|
||||||
|
else if spec.type == "builtin-url" then fetch_builtin-url name
|
||||||
|
else
|
||||||
|
abort "ERROR: niv spec ${name} has unknown type ${builtins.toJSON spec.type}";
|
||||||
|
|
||||||
|
# If the environment variable NIV_OVERRIDE_${name} is set, then use
|
||||||
|
# the path directly as opposed to the fetched source.
|
||||||
|
replace = name: drv:
|
||||||
|
let
|
||||||
|
saneName = stringAsChars (c: if isNull (builtins.match "[a-zA-Z0-9]" c) then "_" else c) name;
|
||||||
|
ersatz = builtins.getEnv "NIV_OVERRIDE_${saneName}";
|
||||||
|
in
|
||||||
|
if ersatz == "" then drv else
|
||||||
|
# this turns the string into an actual Nix path (for both absolute and
|
||||||
|
# relative paths)
|
||||||
|
if builtins.substring 0 1 ersatz == "/" then /. + ersatz else /. + builtins.getEnv "PWD" + "/${ersatz}";
|
||||||
|
|
||||||
|
# Ports of functions for older nix versions
|
||||||
|
|
||||||
|
# a Nix version of mapAttrs if the built-in doesn't exist
|
||||||
|
mapAttrs = builtins.mapAttrs or (
|
||||||
|
f: set: with builtins;
|
||||||
|
listToAttrs (map (attr: { name = attr; value = f attr set.${attr}; }) (attrNames set))
|
||||||
|
);
|
||||||
|
|
||||||
|
# https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/lists.nix#L295
|
||||||
|
range = first: last: if first > last then [] else builtins.genList (n: first + n) (last - first + 1);
|
||||||
|
|
||||||
|
# https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L257
|
||||||
|
stringToCharacters = s: map (p: builtins.substring p 1 s) (range 0 (builtins.stringLength s - 1));
|
||||||
|
|
||||||
|
# https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L269
|
||||||
|
stringAsChars = f: s: concatStrings (map f (stringToCharacters s));
|
||||||
|
concatMapStrings = f: list: concatStrings (map f list);
|
||||||
|
concatStrings = builtins.concatStringsSep "";
|
||||||
|
|
||||||
|
# https://github.com/NixOS/nixpkgs/blob/8a9f58a375c401b96da862d969f66429def1d118/lib/attrsets.nix#L331
|
||||||
|
optionalAttrs = cond: as: if cond then as else {};
|
||||||
|
|
||||||
|
# fetchTarball version that is compatible between all the versions of Nix
|
||||||
|
builtins_fetchTarball = { url, name ? null, sha256 }@attrs:
|
||||||
|
let
|
||||||
|
inherit (builtins) lessThan nixVersion fetchTarball;
|
||||||
|
in
|
||||||
|
if lessThan nixVersion "1.12" then
|
||||||
|
fetchTarball ({ inherit url; } // (optionalAttrs (!isNull name) { inherit name; }))
|
||||||
|
else
|
||||||
|
fetchTarball attrs;
|
||||||
|
|
||||||
|
# fetchurl version that is compatible between all the versions of Nix
|
||||||
|
builtins_fetchurl = { url, name ? null, sha256 }@attrs:
|
||||||
|
let
|
||||||
|
inherit (builtins) lessThan nixVersion fetchurl;
|
||||||
|
in
|
||||||
|
if lessThan nixVersion "1.12" then
|
||||||
|
fetchurl ({ inherit url; } // (optionalAttrs (!isNull name) { inherit name; }))
|
||||||
|
else
|
||||||
|
fetchurl attrs;
|
||||||
|
|
||||||
|
# Create the final "sources" from the config
|
||||||
|
mkSources = config:
|
||||||
|
mapAttrs (
|
||||||
|
name: spec:
|
||||||
|
if builtins.hasAttr "outPath" spec
|
||||||
|
then abort
|
||||||
|
"The values in sources.json should not have an 'outPath' attribute"
|
||||||
|
else
|
||||||
|
spec // { outPath = replace name (fetch config.pkgs name spec); }
|
||||||
|
) config.sources;
|
||||||
|
|
||||||
|
# The "config" used by the fetchers
|
||||||
|
mkConfig =
|
||||||
|
{ sourcesFile ? if builtins.pathExists ./sources.json then ./sources.json else null
|
||||||
|
, sources ? if isNull sourcesFile then {} else builtins.fromJSON (builtins.readFile sourcesFile)
|
||||||
|
, system ? builtins.currentSystem
|
||||||
|
, pkgs ? mkPkgs sources system
|
||||||
|
}: rec {
|
||||||
|
# The sources, i.e. the attribute set of spec name to spec
|
||||||
|
inherit sources;
|
||||||
|
|
||||||
|
# The "pkgs" (evaluated nixpkgs) to use for e.g. non-builtin fetchers
|
||||||
|
inherit pkgs;
|
||||||
|
};
|
||||||
|
|
||||||
|
in
|
||||||
|
mkSources (mkConfig {}) // { __functor = _: settings: mkSources (mkConfig settings); }
|
0
peerix/__init__.py
Normal file
0
peerix/__init__.py
Normal file
13
peerix/__main__.py
Normal file
13
peerix/__main__.py
Normal file
|
@ -0,0 +1,13 @@
|
||||||
|
import asyncio
|
||||||
|
import uvloop
|
||||||
|
from hypercorn import Config
|
||||||
|
from hypercorn.asyncio import serve
|
||||||
|
|
||||||
|
from peerix.app import app
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
uvloop.install()
|
||||||
|
config = Config()
|
||||||
|
config.bind = ["0.0.0.0:12304"]
|
||||||
|
asyncio.run(serve(app, config))
|
71
peerix/app.py
Normal file
71
peerix/app.py
Normal file
|
@ -0,0 +1,71 @@
|
||||||
|
import asyncio
|
||||||
|
import contextlib
|
||||||
|
|
||||||
|
from starlette.requests import Request
|
||||||
|
from starlette.responses import Response, StreamingResponse
|
||||||
|
from starlette.applications import Starlette
|
||||||
|
|
||||||
|
from peerix.local import local
|
||||||
|
from peerix.remote import remote
|
||||||
|
from peerix.prefix import PrefixStore
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
@contextlib.asynccontextmanager
|
||||||
|
async def _setup_stores(local_port: int):
|
||||||
|
global l_access, r_access
|
||||||
|
async with local() as l:
|
||||||
|
l_access = PrefixStore("local/nar", l)
|
||||||
|
lp = PrefixStore("local", l)
|
||||||
|
async with remote(lp, local_port, "0.0.0.0") as r:
|
||||||
|
r_access = PrefixStore("remote", r)
|
||||||
|
yield
|
||||||
|
setup_store = _setup_stores(12304)
|
||||||
|
|
||||||
|
|
||||||
|
app = Starlette()
|
||||||
|
|
||||||
|
@app.on_event("startup")
|
||||||
|
async def _setup_stores_init():
|
||||||
|
await setup_store.__aenter__()
|
||||||
|
|
||||||
|
@app.on_event("shutdown")
|
||||||
|
async def _setup_stores_deinit():
|
||||||
|
await setup_store.__aexit__(None, None, None)
|
||||||
|
|
||||||
|
|
||||||
|
@app.route("/nix-cache-info")
|
||||||
|
async def cache_info(_: Request) -> Response:
|
||||||
|
ci = await l_access.cache_info()
|
||||||
|
ci = ci._replace(priority=20)
|
||||||
|
return Response(content=ci.dump())
|
||||||
|
|
||||||
|
|
||||||
|
@app.route("/{hash:str}.narinfo")
|
||||||
|
async def narinfo(req: Request) -> Response:
|
||||||
|
if req.client.host != "127.0.0.1":
|
||||||
|
return Response(content="Permission denied.", status_code=403)
|
||||||
|
|
||||||
|
ni = await r_access.narinfo(req.path_params["hash"])
|
||||||
|
if ni is None:
|
||||||
|
return Response(content="Not found", status_code=404)
|
||||||
|
|
||||||
|
return Response(content=ni.dump(), status_code=200, media_type="text/x-nix-narinfo")
|
||||||
|
|
||||||
|
@app.route("/local/{hash:str}.narinfo")
|
||||||
|
async def access_narinfo(req: Request) -> Response:
|
||||||
|
ni = await l_access.narinfo(req.path_params["hash"])
|
||||||
|
if ni is None:
|
||||||
|
return Response(content="Not found", status_code=404)
|
||||||
|
|
||||||
|
return Response(content=ni.dump(), status_code=200, media_type="text/x-nix-narinfo")
|
||||||
|
|
||||||
|
|
||||||
|
@app.route("/local/nar/{path:str}")
|
||||||
|
async def push_nar(req: Request) -> Response:
|
||||||
|
return StreamingResponse(l_access.nar(f"local/nar/{req.path_params['path']}"), media_type="text/plain")
|
||||||
|
|
||||||
|
|
||||||
|
@app.route("/remote/{path:path}")
|
||||||
|
async def pull_nar(req: Request) -> Response:
|
||||||
|
return StreamingResponse(l_access.nar(f"remote/{req.path_params['path']}"), media_type="text/plain")
|
106
peerix/local.py
Normal file
106
peerix/local.py
Normal file
|
@ -0,0 +1,106 @@
|
||||||
|
import typing as t
|
||||||
|
|
||||||
|
import contextlib
|
||||||
|
import subprocess
|
||||||
|
import tempfile
|
||||||
|
import asyncio
|
||||||
|
import shutil
|
||||||
|
import base64
|
||||||
|
import sys
|
||||||
|
import os
|
||||||
|
|
||||||
|
import aiohttp
|
||||||
|
|
||||||
|
from peerix.store import NarInfo, CacheInfo, Store
|
||||||
|
|
||||||
|
|
||||||
|
nix_serve = shutil.which("nix-serve")
|
||||||
|
if nix_serve is None:
|
||||||
|
raise RuntimeError("nix-serve is not installed.")
|
||||||
|
|
||||||
|
nix = shutil.which("nix")
|
||||||
|
if nix is None:
|
||||||
|
raise RuntimeError("nix is not installed.")
|
||||||
|
|
||||||
|
assert nix_serve is not None
|
||||||
|
assert nix is not None
|
||||||
|
|
||||||
|
|
||||||
|
class LocalStore(Store):
|
||||||
|
|
||||||
|
def __init__(self, session: aiohttp.ClientSession):
|
||||||
|
self.session = session
|
||||||
|
self._cache: t.Optional[CacheInfo] = None
|
||||||
|
|
||||||
|
async def cache_info(self) -> CacheInfo:
|
||||||
|
if self._cache is None:
|
||||||
|
async with self.session.get("http://_/nix-cache-info") as resp:
|
||||||
|
storeDir = ""
|
||||||
|
wantMassQuery = -1
|
||||||
|
priority = 50
|
||||||
|
|
||||||
|
for line in (await resp.text()).splitlines():
|
||||||
|
k, v = line.split(":", 1)
|
||||||
|
v = v.strip()
|
||||||
|
k = k.strip()
|
||||||
|
|
||||||
|
if k == "StoreDir":
|
||||||
|
storeDir = v
|
||||||
|
elif k == "WantMassQuery":
|
||||||
|
wantMassQuery = int(v)
|
||||||
|
elif k == "Priority":
|
||||||
|
priority = int(v)
|
||||||
|
|
||||||
|
self._cache = CacheInfo(storeDir, wantMassQuery, priority)
|
||||||
|
|
||||||
|
return self._cache
|
||||||
|
|
||||||
|
|
||||||
|
async def narinfo(self, hsh: str) -> t.Optional[NarInfo]:
|
||||||
|
async with self.session.get(f"http://_/{hsh}.narinfo") as resp:
|
||||||
|
if resp.status == 404:
|
||||||
|
return None
|
||||||
|
info = NarInfo.parse(await resp.text())
|
||||||
|
return info._replace(url=base64.b64encode(info.storePath.encode("utf-8")).replace(b"/", b"_").decode("ascii"))
|
||||||
|
|
||||||
|
async def nar(self, sp: str) -> t.AsyncIterable[bytes]:
|
||||||
|
path = base64.b64decode(sp.replace("_", "/")).decode("utf-8")
|
||||||
|
if not path.startswith((await self.cache_info()).storeDir):
|
||||||
|
raise FileNotFoundError()
|
||||||
|
|
||||||
|
process = await asyncio.create_subprocess_exec(
|
||||||
|
nix, "dump-path", "--", path,
|
||||||
|
stdout=subprocess.PIPE,
|
||||||
|
stderr=subprocess.DEVNULL,
|
||||||
|
stdin=subprocess.DEVNULL,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert process.stdout is not None
|
||||||
|
while not process.stdout.at_eof():
|
||||||
|
yield await process.stdout.read(10*1024*1024)
|
||||||
|
|
||||||
|
|
||||||
|
@contextlib.asynccontextmanager
|
||||||
|
async def local():
|
||||||
|
with tempfile.TemporaryDirectory() as tmpdir:
|
||||||
|
sock = f"{tmpdir}/server.sock"
|
||||||
|
process = await asyncio.create_subprocess_exec(
|
||||||
|
nix_serve, "--listen", sock,
|
||||||
|
stdin=subprocess.DEVNULL,
|
||||||
|
stdout=subprocess.DEVNULL,
|
||||||
|
stderr=sys.stderr
|
||||||
|
)
|
||||||
|
for _ in range(10):
|
||||||
|
if os.path.exists(sock):
|
||||||
|
break
|
||||||
|
await asyncio.sleep(1)
|
||||||
|
else:
|
||||||
|
raise RuntimeError("Failed to start up local store.")
|
||||||
|
|
||||||
|
try:
|
||||||
|
connector = aiohttp.UnixConnector(sock)
|
||||||
|
async with aiohttp.ClientSession(connector_owner=True, connector=connector) as session:
|
||||||
|
yield LocalStore(session)
|
||||||
|
finally:
|
||||||
|
process.terminate()
|
||||||
|
|
25
peerix/prefix.py
Normal file
25
peerix/prefix.py
Normal file
|
@ -0,0 +1,25 @@
|
||||||
|
import typing as t
|
||||||
|
from peerix.store import NarInfo, Store
|
||||||
|
|
||||||
|
|
||||||
|
class PrefixStore(Store):
|
||||||
|
def __init__(self, prefix: str, backend: Store):
|
||||||
|
self.backend = backend
|
||||||
|
self.prefix = prefix
|
||||||
|
|
||||||
|
async def cache_info(self):
|
||||||
|
return await self.backend.cache_info()
|
||||||
|
|
||||||
|
async def narinfo(self, hsh: str) -> t.Optional[NarInfo]:
|
||||||
|
info = await self.backend.narinfo(hsh)
|
||||||
|
if info is None:
|
||||||
|
return None
|
||||||
|
return info._replace(url=f"{self.prefix}/{info.url}")
|
||||||
|
|
||||||
|
async def nar(self, path: str) -> t.AsyncIterable[bytes]:
|
||||||
|
if not path.startswith(self.prefix + "/"):
|
||||||
|
raise FileNotFoundError("Not found.")
|
||||||
|
|
||||||
|
async for chunk in self.backend.nar(path[len(self.prefix)+1:]):
|
||||||
|
yield chunk
|
||||||
|
|
108
peerix/remote.py
Normal file
108
peerix/remote.py
Normal file
|
@ -0,0 +1,108 @@
|
||||||
|
import typing as t
|
||||||
|
|
||||||
|
import socket
|
||||||
|
import asyncio
|
||||||
|
import contextlib
|
||||||
|
|
||||||
|
import aiohttp
|
||||||
|
|
||||||
|
|
||||||
|
from peerix.store import NarInfo, Store
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
class DiscoveryProtocol(asyncio.DatagramProtocol, Store):
|
||||||
|
idx: int
|
||||||
|
transport: asyncio.DatagramTransport
|
||||||
|
waiters: t.Dict[int, asyncio.Future]
|
||||||
|
store: Store
|
||||||
|
session: aiohttp.ClientSession
|
||||||
|
local_port: int
|
||||||
|
|
||||||
|
def __init__(self, store: Store, session: aiohttp.ClientSession, local_port: int):
|
||||||
|
self.idx = 0
|
||||||
|
self.waiters = {}
|
||||||
|
self.store = store
|
||||||
|
self.session = session
|
||||||
|
self.local_port = local_port
|
||||||
|
|
||||||
|
def connection_made(self, transport):
|
||||||
|
self.transport = transport
|
||||||
|
|
||||||
|
def datagram_received(self, data: bytes, addr: None) -> None:
|
||||||
|
# 0 => Response to a command of mine.
|
||||||
|
if data[0] == 1:
|
||||||
|
idx = int.from_bytes(data[1:5], "big")
|
||||||
|
if idx not in self.waiters:
|
||||||
|
return
|
||||||
|
|
||||||
|
self.waiters[idx].set_result((data[6:].decode("utf-8"), addr))
|
||||||
|
|
||||||
|
# 1 => Request from another server.
|
||||||
|
elif data[0] == 0:
|
||||||
|
asyncio.create_task(self.respond(data, addr))
|
||||||
|
|
||||||
|
def stop(self):
|
||||||
|
self.transport.close()
|
||||||
|
|
||||||
|
async def cache_info(self):
|
||||||
|
return await self.store.cache_info()
|
||||||
|
|
||||||
|
async def respond(self, data: bytes, addr: None) -> None:
|
||||||
|
hsh = data[6:].decode("utf-8")
|
||||||
|
print("Got request from {addr[0]}:{addr[1]} for {hsh}")
|
||||||
|
narinfo = await self.store.narinfo(hsh)
|
||||||
|
if narinfo is None:
|
||||||
|
return
|
||||||
|
self.transport.sendto(b"\x01" + data[1:5] + self.local_port.to_bytes(4, "big") + narinfo.url.encode("utf-8"))
|
||||||
|
|
||||||
|
async def narinfo(self, hsh: str) -> t.Optional[NarInfo]:
|
||||||
|
fut = asyncio.get_running_loop().create_future()
|
||||||
|
self.idx = (idx := self.idx)+1
|
||||||
|
self.waiters[idx] = fut
|
||||||
|
fut.add_done_callback(lambda _: self.waiters.pop(idx, None))
|
||||||
|
print(f"Requesting {hsh} from direct local network.")
|
||||||
|
self.transport.sendto(b"".join([b"\x00", idx.to_bytes(4, "big"), hsh.encode("utf-8")]), ("255.255.255.255", self.local_port))
|
||||||
|
try:
|
||||||
|
data, addr = await asyncio.wait_for(fut, 0.5)
|
||||||
|
except asyncio.TimeoutError:
|
||||||
|
print(f"No response for {hsh}")
|
||||||
|
return None
|
||||||
|
|
||||||
|
port = int.from_bytes(data[0:4], "big")
|
||||||
|
url = data[5:].decode("utf-8")
|
||||||
|
|
||||||
|
print(f"{addr[0]}:{addr[1]} responded for {hsh} with http://{addr[0]}:{port}/{url}")
|
||||||
|
|
||||||
|
async with self.session.get(f"http://{addr[0]}:{port}/{url}") as resp:
|
||||||
|
if resp.status != 200:
|
||||||
|
return
|
||||||
|
info = NarInfo.parse(await resp.text())
|
||||||
|
|
||||||
|
return info._replace(url = f"{addr[0]}/{port}/{info.url}")
|
||||||
|
|
||||||
|
async def nar(self, sp: str) -> t.AsyncIterable[bytes]:
|
||||||
|
addr1, addr2, p = sp.split("/", 2)
|
||||||
|
async with self.session.get(f"http://{addr1}:{addr2}/{p}") as resp:
|
||||||
|
if resp.status != 200:
|
||||||
|
raise FileNotFoundError("Not found.")
|
||||||
|
content = resp.content
|
||||||
|
while not content.at_eof():
|
||||||
|
yield await content.readany()
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
@contextlib.asynccontextmanager
|
||||||
|
async def remote(store: Store, local_port: int, local_addr: str="0.0.0.0"):
|
||||||
|
protocol: DiscoveryProtocol
|
||||||
|
async with aiohttp.ClientSession() as session:
|
||||||
|
_, protocol = await asyncio.get_running_loop().create_datagram_endpoint(
|
||||||
|
lambda: DiscoveryProtocol(store, session, local_port),
|
||||||
|
local_addr=(local_addr, local_port),
|
||||||
|
family=socket.AF_INET,
|
||||||
|
allow_broadcast=True
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
yield protocol
|
||||||
|
finally:
|
||||||
|
protocol.stop()
|
91
peerix/store.py
Normal file
91
peerix/store.py
Normal file
|
@ -0,0 +1,91 @@
|
||||||
|
import typing as t
|
||||||
|
|
||||||
|
|
||||||
|
class NarInfo(t.NamedTuple):
|
||||||
|
storePath: str
|
||||||
|
url: str
|
||||||
|
compression: t.Literal["none"]
|
||||||
|
narHash: str
|
||||||
|
narSize: int
|
||||||
|
references: t.Sequence[str]
|
||||||
|
deriver: t.Optional[str]
|
||||||
|
signatures: t.Sequence[str]
|
||||||
|
|
||||||
|
def dump(self) -> str:
|
||||||
|
lines = [
|
||||||
|
f"StorePath: {self.storePath}",
|
||||||
|
f"URL: {self.url}",
|
||||||
|
f"Compression: {self.compression}",
|
||||||
|
f"NarHash: {self.narHash}",
|
||||||
|
f"NarSize: {self.narSize}"
|
||||||
|
]
|
||||||
|
if self.references:
|
||||||
|
lines.append(f"References: {' '.join(self.references)}")
|
||||||
|
if self.deriver:
|
||||||
|
lines.append(f"Deriver: {self.deriver} ")
|
||||||
|
for sig in self.signatures:
|
||||||
|
lines.append(f"Sig: {sig}")
|
||||||
|
|
||||||
|
return "\n".join(lines)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def parse(cls, data: str) -> "NarInfo":
|
||||||
|
storePath = ""
|
||||||
|
url = ""
|
||||||
|
compression = "none"
|
||||||
|
narHash = ""
|
||||||
|
narSize = -1
|
||||||
|
references = []
|
||||||
|
deriver = None
|
||||||
|
signatures = []
|
||||||
|
|
||||||
|
for line in data.splitlines():
|
||||||
|
k, v = line.split(":", 1)
|
||||||
|
v = v.strip()
|
||||||
|
k = k.strip()
|
||||||
|
|
||||||
|
if k == "StorePath":
|
||||||
|
storePath = v
|
||||||
|
elif k == "URL":
|
||||||
|
url = v
|
||||||
|
elif k == "Compression" and v == "none":
|
||||||
|
compression = v
|
||||||
|
elif k == "NarHash":
|
||||||
|
narHash = v
|
||||||
|
elif k == "NarSize":
|
||||||
|
narSize = int(v)
|
||||||
|
elif k == "References":
|
||||||
|
references = v.split(" ")
|
||||||
|
elif k == "Deriver":
|
||||||
|
deriver = v
|
||||||
|
elif k == "Sig":
|
||||||
|
signatures.append(v)
|
||||||
|
|
||||||
|
return NarInfo(storePath, url, compression, narHash, narSize, references, deriver, signatures)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
class CacheInfo(t.NamedTuple):
|
||||||
|
storeDir: str
|
||||||
|
wantMassQuery: int
|
||||||
|
priority: int
|
||||||
|
|
||||||
|
def dump(self) -> str:
|
||||||
|
return "\n".join((
|
||||||
|
f"StoreDir: {self.storeDir}",
|
||||||
|
f"WantMassQuery: {self.wantMassQuery}",
|
||||||
|
f"Priority: {self.priority}"
|
||||||
|
))
|
||||||
|
|
||||||
|
|
||||||
|
class Store:
|
||||||
|
|
||||||
|
async def cache_info(self) -> CacheInfo:
|
||||||
|
raise NotImplementedError()
|
||||||
|
|
||||||
|
async def narinfo(self, hsh: str) -> t.Optional[NarInfo]:
|
||||||
|
raise NotImplementedError()
|
||||||
|
|
||||||
|
async def nar(self, url: str) -> t.AsyncIterable[bytes]:
|
||||||
|
raise NotImplementedError()
|
||||||
|
|
4
requirements.txt
Normal file
4
requirements.txt
Normal file
|
@ -0,0 +1,4 @@
|
||||||
|
aiohttp
|
||||||
|
uvloop
|
||||||
|
hypercorn
|
||||||
|
starlette
|
21
setup.py
Normal file
21
setup.py
Normal file
|
@ -0,0 +1,21 @@
|
||||||
|
#setup.py:
|
||||||
|
|
||||||
|
from distutils.core import setup
|
||||||
|
from Cython.Build import cythonize
|
||||||
|
from distutils.extension import Extension
|
||||||
|
|
||||||
|
ext_modules = [
|
||||||
|
Extension(
|
||||||
|
name="peerix._nix",
|
||||||
|
sources=["peerix/_nix.pyx"],
|
||||||
|
language="c++",
|
||||||
|
extra_compile_args=["-std=c++17"],
|
||||||
|
)
|
||||||
|
]
|
||||||
|
|
||||||
|
ext_modules = cythonize(ext_modules)
|
||||||
|
|
||||||
|
setup(
|
||||||
|
name="peerix",
|
||||||
|
ext_modules=ext_modules,
|
||||||
|
)
|
19
shell.nix
Normal file
19
shell.nix
Normal file
|
@ -0,0 +1,19 @@
|
||||||
|
let
|
||||||
|
sources = import ./nix/sources.nix {};
|
||||||
|
in
|
||||||
|
{ pkgs ? import sources.nixpkgs {} }:
|
||||||
|
let
|
||||||
|
mach-nix = import sources.mach-nix {
|
||||||
|
inherit pkgs;
|
||||||
|
};
|
||||||
|
in
|
||||||
|
pkgs.mkShell {
|
||||||
|
buildInputs = with pkgs; [
|
||||||
|
nix-serve
|
||||||
|
niv
|
||||||
|
(mach-nix.mkPython {
|
||||||
|
python = "python39";
|
||||||
|
requirements = (builtins.readFile ./requirements.txt) + "\nipython";
|
||||||
|
})
|
||||||
|
];
|
||||||
|
}
|
Loading…
Reference in a new issue