diff --git a/hosts/packages/huggingface_hub/flake.nix b/hosts/packages/huggingface_hub/flake.nix new file mode 100644 index 0000000..37814d9 --- /dev/null +++ b/hosts/packages/huggingface_hub/flake.nix @@ -0,0 +1,35 @@ +# Template: https://nixos-and-flakes.thiscute.world/development/intro +{ + description = "Humanity's Last Exam - Devshell"; + + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-25.05"; + }; + + outputs = {nixpkgs, ...}: let + system = "x86_64-linux"; + pkgs = import nixpkgs { + inherit system; + }; + python = pkgs.python312.override { + self = python; + packageOverrides = pyfinal: pyprev: { + huggingface-hub = pyfinal.callPackage ./huggingface_hub.nix {}; + hf-xet = pyfinal.callPackage ./hf-xet.nix {}; + }; + }; + in { + devShells."${system}".default = pkgs.mkShell { + packages = [ + (python.withPackages (pypkgs: [ + pypkgs.huggingface-hub + ])) + ]; + + shell = "${pkgs.bash}/bin/bash"; + shellHook = '' + alias hf=huggingface-cli + ''; + }; + }; +} diff --git a/hosts/packages/huggingface_hub/hf-xet.nix b/hosts/packages/huggingface_hub/hf-xet.nix new file mode 100644 index 0000000..0f4c631 --- /dev/null +++ b/hosts/packages/huggingface_hub/hf-xet.nix @@ -0,0 +1,56 @@ +{ + lib, + buildPythonPackage, + fetchFromGitHub, + pkg-config, + rustPlatform, + openssl, +}: +buildPythonPackage rec { + pname = "hf-xet"; + version = "1.1.5"; + pyproject = true; + + src = fetchFromGitHub { + owner = "huggingface"; + repo = "xet-core"; + tag = "v${version}"; + hash = "sha256-udjZcXTH+Mc4Gvj6bSPv1xi4MyXrLeCYav+7CzKWyhY="; + }; + + sourceRoot = "${src.name}/hf_xet"; + + cargoDeps = rustPlatform.fetchCargoVendor { + inherit + pname + version + src + sourceRoot + ; + hash = "sha256-PTzYubJHFvhq6T3314R4aqBAJlwehOqF7SbpLu4Jo6E="; + }; + + nativeBuildInputs = [ + pkg-config + rustPlatform.cargoSetupHook + rustPlatform.maturinBuildHook + ]; + + buildInputs = [ + openssl + ]; + + env.OPENSSL_NO_VENDOR = 1; + + pythonImportsCheck = ["hf_xet"]; + + # No tests (yet?) + doCheck = false; + + meta = { + description = "Xet client tech, used in huggingface_hub"; + homepage = "https://github.com/huggingface/xet-core/tree/main/hf_xet"; + changelog = "https://github.com/huggingface/xet-core/releases/tag/v${version}"; + license = lib.licenses.asl20; + }; +} diff --git a/hosts/packages/huggingface_hub/huggingface_hub.nix b/hosts/packages/huggingface_hub/huggingface_hub.nix new file mode 100644 index 0000000..02b03b7 --- /dev/null +++ b/hosts/packages/huggingface_hub/huggingface_hub.nix @@ -0,0 +1,62 @@ +/* +* WARNING: Just use `pkgs.python312Packages.huggingface-hub` (or change python version) +* WARNING: I didn't realise it existed when I packaged this. +* +* Nix Resources: +* 1. https://wiki.nixos.org/wiki/Python +* 2. https://nixos.org/manual/nixpkgs/unstable/#developing-with-python +* +* Hugging Face Resources: +* 1. https://github.com/huggingface/huggingface_hub +* 2. https://huggingface.co/docs/huggingface_hub/main/en/guides/cli +*/ +{ + lib, + buildPythonPackage, + fetchPypi, + # build time dependencies + setuptools, + # runtime dependencies + filelock, + fsspec, + hf-xet, + pyyaml, + requests, + tqdm, + typing-extensions, +}: +buildPythonPackage rec { + pname = "huggingface_hub"; + version = "0.34.3"; + + src = fetchPypi { + inherit pname version; + hash = "sha256-1YEw/VqnQISAaBR1SRwKvX6DVEIIL7w+9NRbbDn4OFM="; + }; + + pyproject = true; + doCheck = false; # skip unit testing + pythonImportsCheck = ["huggingface_hub"]; + + # buildtime dependencies + build-system = [ + setuptools + ]; + # runtime dependencies + dependencies = [ + filelock + fsspec + hf-xet + pyyaml + requests + tqdm + typing-extensions + ]; + + meta = rec { + description = "The official Python client for the Huggingface Hub."; + homepage = "https://github.com/huggingface/huggingface_hub"; + changelog = "${homepage}/releases/tag/v${version}"; + license = lib.licenses.asl20; # Apache License 2.0 + }; +}