From 5bb174079eecf73d758fe45079cf61ce80b848d1 Mon Sep 17 00:00:00 2001 From: RadinPirouz Date: Sat, 4 Oct 2025 12:40:39 +0330 Subject: [PATCH] gitlab cache added runner config for s3 --- Code-Management/Gitlab/Gitlab-Cache.md | 76 ++++++++++++++++---------- 1 file changed, 48 insertions(+), 28 deletions(-) diff --git a/Code-Management/Gitlab/Gitlab-Cache.md b/Code-Management/Gitlab/Gitlab-Cache.md index b7c4461..79f2932 100644 --- a/Code-Management/Gitlab/Gitlab-Cache.md +++ b/Code-Management/Gitlab/Gitlab-Cache.md @@ -1,21 +1,21 @@ # GitLab CI/CD Cache Guide -GitLab provides a caching system for jobs to speed up pipelines by reusing dependencies like `node_modules`, Ruby gems, and other build artifacts. This guide covers how to configure and use caching effectively. +GitLab CI/CD provides a caching mechanism to speed up pipeline execution by reusing dependencies and build artifacts such as `node_modules`, Ruby gems, or compiled binaries. This guide covers how to configure, optimize, and manage caching effectively. --- -## Key Features +## Key Features of GitLab Cache -* **Cache Storage**: Data is saved in the GitLab Runner and can also be stored in external storage such as S3, MinIO, GCS, etc. -* **Branch-Specific Caching**: Maintain separate caches per branch, job, or stage. -* **Fallbacks**: Jobs can pull cache from other branches if the current branch does not have cache. -* **Simple Cache**: Uses file locks and directory caching for efficiency. +* **Cache Storage**: Cache data is stored in the GitLab Runner and can optionally be persisted in external storage such as S3, MinIO, Google Cloud Storage (GCS), or Azure Blob Storage. +* **Branch-Specific Caching**: Create independent caches per branch, job, or pipeline stage to avoid conflicts. +* **Fallbacks**: Jobs can retrieve cache from other branches if the current branch has no cache yet. +* **Simple & Efficient**: Uses file locks and directory caching to maximize efficiency without complex configurations. --- ## Basic Cache Example -Cache dependencies like Ruby gems and Yarn modules: +A typical caching configuration for Ruby gems and Yarn modules: ```yaml test-job: @@ -39,13 +39,34 @@ test-job: ``` **Behavior**: -If the lock file (`Gemfile.lock` or `yarn.lock`) changes, the cache is updated automatically. +If the lock file (`Gemfile.lock` or `yarn.lock`) changes, the cache is updated automatically, ensuring dependencies remain consistent with the project. --- -## Branch Cache with Fallbacks +## Configuring GitLab Runner to Use S3 Cache -When creating a new branch, it may not have a cache yet. You can pull from the default branch (e.g., `main` or `master`) or a global default cache: +You can configure the GitLab Runner to save cache to S3 storage as follows: + +```toml +[[runners]] + [runners.cache] + Type = "s3" + [runners.cache.s3] + ServerAddress = "s3.example.com" + AccessKey = "access-key" + SecretKey = "secret-key" + BucketName = "runner" + Insecure = true # true = http, false = https + [runners.cache.gcs] + [runners.cache.azure] + [runners.docker] +``` + +--- + +## Branch-Specific Cache with Fallbacks + +When a new branch does not yet have a cache, it can use fallback caches from other branches: ```yaml test-job: @@ -67,7 +88,7 @@ test-job: ## Default Cache Configuration -Define a global cache for all jobs and override it for specific jobs as needed: +Define a global cache and override it per job as necessary: ```yaml default: @@ -82,30 +103,30 @@ default: job: cache: <<: *global_cache - policy: pull # Override policy for this job + policy: pull # Overrides global policy for this job ``` --- ## Cache Policies -| Policy | Description | -| --------- | ------------------------------------------------------ | -| pull | Only fetch the cache, do not update it. | -| pull-push | Fetch the cache and update it after the job completes. | +| Policy | Description | +| --------- | ------------------------------------------------- | +| pull | Fetches cache without updating it | +| pull-push | Fetches cache and updates it after job completion | --- ## Cache Examples -* **Per Job & Branch** +**Per Job & Branch**: ```yaml cache: key: "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG" ``` -* **Per Stage & Branch** +**Per Stage & Branch**: ```yaml cache: @@ -116,7 +137,7 @@ cache: ## Conditional Cache Policy -Update cache only on the default branch; otherwise, just pull: +Update cache only on the default branch while other branches only fetch: ```yaml conditional-policy: @@ -142,7 +163,7 @@ conditional-policy: ## Cache Based on Lock Files -Update cache automatically when `package-lock.json` changes: +Automatically update cache when lock files change, e.g., `package-lock.json`: ```yaml default: @@ -154,16 +175,15 @@ default: - .npm/ ``` -**Tip**: This ensures your npm modules are always synced with the lock file. +**Tip**: Using lock files ensures that cached dependencies are always consistent with the project’s declared versions. --- -## Summary +## Best Practices Summary -* Use **branch-specific keys** for independent caches. -* Use **fallback_keys** to share cache between branches. -* **Policies** control whether cache is updated or only fetched. -* Use **lock files** to trigger cache updates automatically. - -Caching effectively can dramatically reduce build times and maintain clean, efficient pipelines. +* Use **branch-specific keys** for independent caches to avoid conflicts. +* Use **fallback_keys** to share caches across branches. +* Use **cache policies** to control whether cache is updated or only fetched. +* Leverage **lock files** to trigger cache updates automatically. +Proper caching can dramatically reduce build times, improve pipeline efficiency, and ensure consistent builds across branches and environments.