gfal2-util, python3Packages.gfal2-util: init at 1.8.1

Add also passthru.fetchGfal2. This fetcher is currently used only by
passthru.tsets, but capable to be used as a domain-specific
real-world fetcher.
This commit is contained in:
Yueh-Shun Li 2023-11-13 06:55:08 +00:00
parent 9cbe717fc2
commit fc61cff2d0
6 changed files with 182 additions and 0 deletions

View file

@ -0,0 +1,2 @@
{ python3Packages }:
with python3Packages; toPythonApplication gfal2-util

View file

@ -0,0 +1,103 @@
{ lib
, buildPythonPackage
, callPackage
, fetchFromGitHub
, runCommandLocal
# Build inputs
, gfal2-python
# For tests
, xrootd # pkgs.xrootd
}:
(buildPythonPackage rec {
pname = "gfal2-util";
version = "1.8.1";
src = fetchFromGitHub {
owner = "cern-fts";
repo = "gfal2-util";
rev = "v${version}";
hash = "sha256-3JbJgKD17aYkrB/aaww7IQU8fLFrTCh868KWlLPxmlk=";
};
# Replace the ad-hoc python executable finding
# and change the shebangs from `#!/bin/sh` to `#!/usr/bin/env python`
# for fixup phase to work correctly.
postPatch = ''
for script in src/gfal-*; do
patch "$script" ${./gfal-util-script.patch}
done
'';
propagatedBuildInputs = [
gfal2-python
];
pythonImportsCheck = [
"gfal2_util"
];
meta = with lib; {
description = "CLI for gfal2";
homepage = "https://github.com/cern-fts/gfal2-utils";
license = licenses.asl20;
maintainers = with maintainers; [ ShamrockLee ];
};
}).overrideAttrs (finalAttrs: previousAttrs: lib.recursiveUpdate previousAttrs {
passthru = {
inherit (gfal2-python) gfal2;
fetchGfal2 = lib.makeOverridable (callPackage ./fetchgfal2.nix { gfal2-util = finalAttrs.finalPackage; });
# With these functionality tests, it should be safe to merge version bumps once all the tests are passed.
tests =
let
# Use the the bin output hash of gfal2-util as version to ensure that
# the test gets rebuild everytime gfal2-util gets rebuild
versionFODTests = finalAttrs.version + "-" + lib.substring (lib.stringLength builtins.storeDir + 1) 32 "${self}";
self = finalAttrs.finalPackage;
in
lib.optionalAttrs gfal2-python.gfal2.enablePluginStatus.xrootd (
let
# Test against a real-world dataset from CERN Open Data
# borrowed from `xrootd.tests`.
urlTestFile = xrootd.tests.test-xrdcp.url;
hashTestFile = xrootd.tests.test-xrdcp.outputHash;
urlTestDir = dirOf urlTestFile;
in
{
test-copy-file-xrootd = finalAttrs.passthru.fetchGfal2 {
url = urlTestFile;
hash = hashTestFile;
extraGfalCopyFlags = [ "--verbose" ];
pname = "gfal2-util-test-copy-file-xrootd";
version = versionFODTests;
allowSubstitutes = false;
};
test-copy-dir-xrootd = finalAttrs.passthru.fetchGfal2 {
url = urlTestDir;
hash = "sha256-vOahIhvx1oE9sfkqANMGUvGeLHS737wyfYWo4rkvrxw=";
recursive = true;
extraGfalCopyFlags = [ "--verbose" ];
pname = "gfal2-util-test-copy-dir-xrootd";
version = versionFODTests;
allowSubstitutes = false;
};
test-ls-dir-xrootd = (runCommandLocal "test-gfal2-util-ls-dir-xrootd" { } ''
set -eu -o pipefail
gfal-ls "$url" | grep "$baseNameExpected" | tee "$out"
'').overrideAttrs (finalAttrs: previousAttrs: {
pname = previousAttrs.name;
version = versionFODTests;
name = "${finalAttrs.pname}-${finalAttrs.version}";
nativeBuildInputs = [ self ];
url = urlTestDir;
baseNameExpected = baseNameOf urlTestFile;
outputHashMode = "flat";
outputHashAlgo = "sha256";
outputHash = builtins.hashString finalAttrs.outputHashAlgo (finalAttrs.baseNameExpected + "\n");
});
}
);
};
})

View file

@ -0,0 +1,48 @@
{ lib
, runCommandLocal
, gfal2-util
}:
# `url` and `urls` should only be overriden via `<pkg>.override`, but not `<pkg>.overrideAttrs`.
{ name ? ""
, pname ? ""
, version ? ""
, urls ? [ ]
, url ? if urls == [ ] then abort "Expect either non-empty `urls` or `url`" else lib.head urls
, hash ? lib.fakeHash
, recursive ? false
, intermediateDestUrls ? [ ]
, extraGfalCopyFlags ? [ ]
, allowSubstitutes ? true
}:
(runCommandLocal name { } ''
for u in "''${urls[@]}"; do
gfal-copy "''${gfalCopyFlags[@]}" "$u" "''${intermediateDestUrls[@]}" "$out"
ret="$?"
(( ret )) && break
done
if (( ret )); then
echo "gfal-copy failed trying to download from any of the urls" >&2
exit "$ret"
fi
'').overrideAttrs (finalAttrs: previousAttrs:
{
__structuredAttrs = true;
inherit allowSubstitutes;
nativeBuildInputs = [ gfal2-util ];
outputHashAlgo = null;
outputHashMode = if finalAttrs.recursive then "recursive" else "flat";
outputHash = hash;
inherit url;
urls = if urls == [ ] then lib.singleton url else urls;
gfalCopyFlags = extraGfalCopyFlags
++ lib.optional finalAttrs.recursive "--recursive"
;
inherit recursive intermediateDestUrls;
} // (if (pname != "" && version != "") then {
inherit pname version;
name = "${finalAttrs.pname}-${finalAttrs.version}";
} else {
name = if (name != "") then name else (baseNameOf url);
}))

View file

@ -0,0 +1,19 @@
--- a/gfal-SCRIPT_NAME 2022-11-17 00:00:00.000000000 +0000
+++ b/gfal-SCRIPT_NAME 2022-11-17 00:00:00.000000000 +0000
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright (c) 2022 CERN
@@ -15,10 +15,2 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-
-# Execute script content with first python interpreter found:
-# * GFAL_PYTHONBIN environment variable
-# * python on the PATH if import gfal2, gfal2_util succeeds
-# * python3 on the PATH if import gfal2, gfal2_util succeeds
-# * python2 on the PATH if import gfal2, gfal2_util succeeds
-# * /usr/bin/python
-"exec" "$( check_interpreter() { unalias $1 2> /dev/null; unset $1; GFAL_PYTHONBIN=$(command -v $1); [ $GFAL_PYTHONBIN ] && $GFAL_PYTHONBIN -c 'import gfal2, gfal2_util' > /dev/null 2>&1 && { echo $GFAL_PYTHONBIN; unset GFAL_PYTHONBIN; }; }; [ $GFAL_PYTHONBIN ] && echo $GFAL_PYTHONBIN || check_interpreter python || check_interpreter python3 || check_interpreter python2 || echo /usr/bin/python )" "-u" "-Wignore" "$0" "$@"

View file

@ -1834,6 +1834,12 @@ with pkgs;
github-copilot-cli = callPackage ../tools/misc/github-copilot-cli { };
# This is to workaround gfal2-python broken against Python 3.12 or later.
# TODO: Remove these lines after solving the breakage.
gfal2-util = callPackage ../by-name/gf/gfal2-util/package.nix (lib.optionalAttrs python3Packages.gfal2-python.meta.broken {
python3Packages = python311Packages;
});
gfshare = callPackage ../tools/security/gfshare { };
gh-actions-cache = callPackage ../tools/misc/gh-actions-cache { };

View file

@ -4687,6 +4687,10 @@ self: super: with self; {
gfal2-python = callPackage ../development/python-modules/gfal2-python { };
gfal2-util = callPackage ../development/python-modules/gfal2-util {
inherit (pkgs) xrootd;
};
gflags = callPackage ../development/python-modules/gflags { };
gflanguages = callPackage ../development/python-modules/gflanguages { };