Merge pull request #5112 from kaspar030/use_git_cache

make: pkg: introduce git-cache
2024-12-29 04:50:03 +01:00 · 2016-03-21 22:32:33 +01:00 · 2016-03-21 22:32:33 +01:00 · 16a3f131a3
commit 16a3f131a3
parent b523e8e04b 812da2f0e7
13 changed files with 155 additions and 26 deletions
--- a/Makefile.include
+++ b/Makefile.include
@ -20,6 +20,8 @@ RIOTPKG ?= $(RIOTBASE)/pkg
 RIOTPROJECT ?= $(shell git rev-parse --show-toplevel 2>/dev/null || pwd)
 RIOTPROJECT := $(abspath $(RIOTPROJECT))

+GITCACHE:=$(RIOTBASE)/dist/tools/git/git-cache
+
 # Include Docker settings near the top because we need to build the environment
 # command line before some of the variable origins are overwritten below when
 # using abspath, strip etc.
--- a/Makefile.vars
+++ b/Makefile.vars
@ -43,6 +43,7 @@ export SIZE                  # The command to read to size of the ELF sections.
 export UNDEF                 # Set by the BOARD's and CPU's Makefile.include, this contains object files with must not be used in the ELFFILE even if the if no call to the functions.
 export WERROR                # Treat all compiler warnings as errors if set to 1 (see -Werror flag in GCC manual)

+export GITCACHE              # path to git-cache executable
 export FLASHER               # The command to call on "make flash".
 export FFLAGS                # The parameters to supply to FLASHER.
 export TERMPROG              # The command to call on "make term".
--- a/dist/tools/git/README.md
+++ b/dist/tools/git/README.md
@ -0,0 +1,21 @@
+# Overview
+
+This directory contains some git tools used by RIOT's build system
+
+## git-cache
+
+Simple git caching script, from https://github.com/kaspar030/git-cache
+If git-cache is unconfigured, the script pulls from the given remote location.
+
+In order to set up the cache, do:
+
+- install the git-cache binary into path.
+  This will make the script available as "git cache ...".
+  Alternatively, directly execute it.
+- run "git cache init", which initializes a git cache in ${HOME}/.gitcache.
+  The used path can be overridden using the "GIT_CACHE_DIR" environment
+  variable.
+  The cache repository will be used to cache multiple remote repositories.
+- add a repository to the cache: "git cache add \<name\> \<URL\>
+- whenever needed (at least once after adding a repository),
+  run "git cache update"
--- a/dist/tools/git/git-cache
+++ b/dist/tools/git/git-cache
@ -0,0 +1,121 @@
+#!/bin/sh
+
+git_cache() {
+    git -C "${GIT_CACHE_DIR}" $*
+}
+
+init() {
+    set -ex
+    test -d "${GIT_CACHE_DIR}/.git" || {
+        mkdir -p "${GIT_CACHE_DIR}"
+
+        git_cache init --bare
+        git_cache config core.compression 1
+    }
+    set +ex
+}
+
+add() {
+    set -ex
+    git_cache remote add $1 $2
+    set +ex
+}
+
+update() {
+    set -ex
+    local REMOTE=${1:---all}
+    git_cache fetch $REMOTE
+    set +ex
+}
+
+list() {
+    local REMOTES="$(git_cache remote show)"
+    for remote in $REMOTES; do
+        echo "${remote}: $(git_cache remote get-url $remote)"
+    done
+}
+
+drop() {
+    set -ex
+    local REMOTE=${1}
+    [ -z "$REMOTE" ] && {
+        echo "usage: git cache drop <name>"
+        exit 1
+    }
+    git_cache remote remove $REMOTE
+    set +ex
+}
+
+_check_commit() {
+    git_cache cat-file -e ${1}^{commit}
+}
+
+clone() {
+    set -ex
+    local REMOTE="${1}"
+    local SHA1="${2}"
+    local REMOTE_NAME="$(basename $REMOTE)"
+    local TARGET_PATH="${3:-${REMOTE_NAME}}"
+
+    if _check_commit $2 2>&1; then
+        git init "${TARGET_PATH}"
+        git_cache tag commit$SHA1 $SHA1 || true # ignore possibly already existing tag
+        git -C "${TARGET_PATH}" fetch --depth=1 "${GIT_CACHE_DIR}" refs/tags/commit$SHA1
+        git -C "${TARGET_PATH}" checkout FETCH_HEAD
+    else
+        git clone "${REMOTE}" "${TARGET_PATH}"
+        git -C "${TARGET_PATH}" checkout $SHA1
+    fi
+    set +ex
+}
+
+usage() {
+    echo "git cache uses a bare git repository containing all objects from multiple"
+    echo "upstream git repositories."
+    echo ""
+    echo "usage:"
+    echo ""
+    echo "    git cache init                initialize git cache"
+    echo "    git cache add <name> <url>    add repository <url> with name <name>"
+    echo "    git cache list                list cached repositories"
+    echo "    git cache drop <name>         drop repo from cache"
+    echo "    git cache update [<name>]     fetch repo named <name> (or all)"
+    echo "    git cache clone <url> <SHA1>  clone repository <url> from cache"
+    echo "    git cache show-path           print's the path that can be used as "
+    echo "                                  '--reference' parameter"
+    echo ""
+    echo "To retrieve objects from cache (will use remote repository if needed):"
+    echo '    git clone --reference $(git cache show-path) <repo>'
+}
+
+ACTION=$1
+shift
+
+export GIT_CACHE_DIR=${GIT_CACHE_DIR:-${HOME}/.gitcache}
+
+case $ACTION in
+    init)
+        init $*
+        ;;
+    add)
+        add $*
+        ;;
+    update)
+        update $*
+        ;;
+    list)
+        list $*
+        ;;
+    drop)
+        drop $*
+        ;;
+    show-path)
+        echo ${GIT_CACHE_DIR}
+        ;;
+    clone)
+        clone $*
+        ;;
+    *)
+        usage
+        ;;
+esac
--- a/pkg/ccn-lite/Makefile
+++ b/pkg/ccn-lite/Makefile
@ -16,10 +16,7 @@ all: $(PKG_DIR)/Makefile
 $(PKG_DIR)/Makefile: $(PKG_DIR)/.git/config

 $(PKG_DIR)/.git/config:
-	test -d "$(PKG_DIR)" || git clone "$(PKG_URL)" "$(PKG_DIR)"; \
-		cd "$(PKG_DIR)" && \
-		git remote set-url origin "$(PKG_URL)" && \
-		git fetch && git checkout -f "$(PKG_VERSION)"
+	test -d "$(PKG_DIR)" || $(GITCACHE) clone "$(PKG_URL)" "$(PKG_VERSION)" "$(PKG_DIR)"

 clean::
 	@echo "Cleaning up CCN-Lite package..."
--- a/pkg/cmsis-dsp/Makefile
+++ b/pkg/cmsis-dsp/Makefile
@ -16,12 +16,7 @@ $(PKG_DIR)/Makefile: $(PKG_DIR)/.git/config
 	@

 $(PKG_DIR)/.git/config:
-	test -d "$(PKG_DIR)" || git clone "$(PKG_URL)" "$(PKG_DIR)"; \
-		cd "$(PKG_DIR)" && git checkout -f "$(PKG_VERSION)"
-
-$(CURDIR)/$(PKG_NAME) $(PKG_NAME):
-	# Get PKG_VERSION of package from PKG_URL
-	git clone '$(PKG_URL)' '$(PKG_NAME)' && cd '$(PKG_NAME)' && git reset --hard '$(PKG_VERSION)'
+	test -d "$(PKG_DIR)" || $(GITCACHE) clone "$(PKG_URL)" "$(PKG_VERSION)" "$(PKG_DIR)"

 clean::
 	@echo "Cleaning up $(PKG_NAME) package..."
--- a/pkg/libcoap/Makefile
+++ b/pkg/libcoap/Makefile
@ -19,8 +19,7 @@ $(PKG_DIR)/Makefile: $(PKG_DIR)/.git/config
 	cd "$(PKG_DIR)" && git am --ignore-whitespace "$(CURDIR)"/*.patch

 $(PKG_DIR)/.git/config:
-	test -d "$(PKG_DIR)" || git clone "$(PKG_URL)" "$(PKG_DIR)"; \
-		cd "$(PKG_DIR)" && git checkout -f "$(PKG_VERSION)"
+	test -d "$(PKG_DIR)" || $(GITCACHE) clone "$(PKG_URL)" "$(PKG_VERSION)" "$(PKG_DIR)"

 clean::
 	@echo "Cleaning up libcoap package..."
--- a/pkg/micro-ecc/Makefile
+++ b/pkg/micro-ecc/Makefile
@ -26,8 +26,7 @@ $(PKG_BUILDDIR)/Makefile: $(PKG_BUILDDIR)

 $(PKG_BUILDDIR):
 	mkdir -p $(BINDIR)/pkg && \
-	git clone $(PKG_URL) $@ && \
-		cd $@ && git reset --hard $(PKG_VERSION)
+	$(GITCACHE) clone "$(PKG_URL)" "$(PKG_VERSION)" "$(PKG_BUILDDIR)"

 clean::
 	rm -Rf $(PKG_BUILDDIR)
--- a/pkg/microcoap/Makefile
+++ b/pkg/microcoap/Makefile
@ -19,8 +19,7 @@ $(PKG_DIR)/Makefile: $(PKG_DIR)/.git/config
 	cd "$(PKG_DIR)" && git am --ignore-whitespace "$(CURDIR)"/*.patch

 $(PKG_DIR)/.git/config:
-	test -d "$(PKG_DIR)" || git clone "$(PKG_URL)" "$(PKG_DIR)"; \
-		cd "$(PKG_DIR)" && git checkout -f "$(PKG_VERSION)"
+	test -d "$(PKG_DIR)" || $(GITCACHE) clone "$(PKG_URL)" "$(PKG_VERSION)" "$(PKG_DIR)"

 clean::
 	@echo "Cleaning up $(PKG_NAME) package..."
--- a/pkg/oonf_api/Makefile
+++ b/pkg/oonf_api/Makefile
@ -22,8 +22,7 @@ $(PKG_DIR)/Makefile: $(PKG_DIR)/.git/config
 	cd "$(PKG_DIR)" && git am --ignore-whitespace "$(CURDIR)"/*.patch

 $(PKG_DIR)/.git/config:
-	test -d "$(PKG_DIR)" || git clone "$(PKG_URL)" "$(PKG_DIR)"; \
-		cd "$(PKG_DIR)" && git checkout -f "$(PKG_VERSION)"
+	test -d "$(PKG_DIR)" || $(GITCACHE) clone "$(PKG_URL)" "$(PKG_VERSION)" "$(PKG_DIR)"

 clean::
 	@echo "Cleaning up oonf_api package..."
--- a/pkg/openwsn/Makefile
+++ b/pkg/openwsn/Makefile
@ -14,8 +14,7 @@ $(PKG_DIR)/Makefile: $(PKG_DIR)/.git/config
 	cd "$(PKG_DIR)" && git am --ignore-whitespace $(CURDIR)/*.patch

 $(PKG_DIR)/.git/config:
-	test -d "$(PKG_DIR)" || git clone "$(PKG_URL)" "$(PKG_DIR)"; \
-		cd "$(PKG_DIR)" && git checkout -f "$(PKG_VERSION)"
+	test -d "$(PKG_DIR)" || $(GITCACHE) clone "$(PKG_URL)" "$(PKG_VERSION)" "$(PKG_DIR)"

 clean::
 	@echo "Cleaning up OpenWSN package..."
@ -29,4 +28,4 @@ distclean::
 	rm -rf "$(PKG_DIR)"

 Makefile.include:
-	@true
+	@true
--- a/pkg/relic/Makefile
+++ b/pkg/relic/Makefile
@ -23,8 +23,7 @@ $(PKG_DIR)/comp-options.cmake: $(PKG_DIR)/.git/config
 $(PKG_DIR)/Makefile: $(PKG_DIR)/comp-options.cmake
 	cd "$(PKG_DIR)" && COMP="$(filter-out -Werror=old-style-definition -Werror=strict-prototypes, $(CFLAGS) ) " cmake -DCMAKE_TOOLCHAIN_FILE=comp-options.cmake -DCHECK=off -DTESTS=0 -DBENCH=0 -DSHLIB=off -Wno-dev $(RELIC_CONFIG_FLAGS) .
 $(PKG_DIR)/.git/config:
-	test -d "$(PKG_DIR)" || git clone "$(PKG_URL)" "$(PKG_DIR)"; \
-		cd "$(PKG_DIR)" && git checkout -f "$(PKG_VERSION)"
+	test -d "$(PKG_DIR)" || $(GITCACHE) clone "$(PKG_URL)" "$(PKG_VERSION)" "$(PKG_DIR)"
 	cd "$(PKG_DIR)" && git am --ignore-whitespace $(CURDIR)/*.patch
 	./fix-util_print_wo_args.sh .
 	./fix-old-style-definitions.sh .
--- a/pkg/wakaama/Makefile
+++ b/pkg/wakaama/Makefile
@ -15,9 +15,7 @@ $(PKG_DIR)/Makefile: $(PKG_TEMP_DIR)/.git/config
 	echo 'include $$(RIOTBASE)/Makefile.base' > $(PKG_DIR)/Makefile

 $(PKG_TEMP_DIR)/.git/config:
-	test -d "$(PKG_TEMP_DIR)" || git clone "$(PKG_URL)" "$(PKG_TEMP_DIR)"; \
-		cd "$(PKG_TEMP_DIR)" && git checkout -f "$(PKG_VERSION)"; \
-		cd "$(PKG_TEMP_DIR)" && git am --ignore-whitespace "$(CURDIR)"/*.patch; \
+	test -d "$(PKG_TEMP_DIR)" || $(GITCACHE) clone "$(PKG_URL)" "$(PKG_VERSION)" "$(PKG_TEMP_DIR)" ; \
 		mkdir -p "$(PKG_DIR)" ; \
 		cp $(PKG_TEMP_DIR)/core/*.c $(PKG_TEMP_DIR)/core/*.h $(PKG_DIR); \
 		cp $(PKG_TEMP_DIR)/core/er-coap-13/*.c $(PKG_TEMP_DIR)/core/er-coap-13/*.h $(PKG_DIR); \