Skip to content

Commit 6cd145a

Browse files
CopilotCopilot
authored
Align GitLab token regexes with TruffleHog and add source provenance (#566)
* Add built-in rules for GitLab token types to detect them regardless of TruffleHog verification
1 parent 12ea000 commit 6cd145a

File tree

3 files changed

+144
-2
lines changed

3 files changed

+144
-2
lines changed

‎pkg/scanner/engine/engine_test.go‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package engine
22

33
import (
44
"fmt"
5+
"strings"
56
"testing"
67
"time"
78

@@ -249,3 +250,76 @@ func TestDeduplicateFindingsWithState_TrimsAtLimit(t *testing.T) {
249250
t.Fatalf("expected state len 500 after trim, got %d", len(newState))
250251
}
251252
}
253+
254+
func TestDetectHits_GitLabTokenDetection(t *testing.T) {
255+
// GitLab tokens should be detected by built-in rules regardless of TruffleHog
256+
// verification status (which only verifies against gitlab.com).
257+
tests := []struct {
258+
name string
259+
text []byte
260+
token string
261+
}{
262+
{
263+
name: "personal access token v2",
264+
text: []byte("export GITLAB_TOKEN=glpat-abcdefghij1234567890"),
265+
token: "glpat-",
266+
},
267+
{
268+
name: "personal access token v2 max length",
269+
text: []byte("export GITLAB_TOKEN=glpat-abcdefghij12345678901x"),
270+
token: "glpat-",
271+
},
272+
{
273+
name: "personal access token v3",
274+
text: []byte("export GITLAB_TOKEN=glpat-abcDEFghij1234567890_-=xyzABC0ab.ab.abc012345"),
275+
token: "glpat-",
276+
},
277+
{
278+
name: "pipeline trigger token",
279+
text: []byte("TRIGGER_TOKEN=glptt-abcdefghij1234567890"),
280+
token: "glptt-",
281+
},
282+
{
283+
name: "deploy token",
284+
text: []byte("DEPLOY_TOKEN=gldt-abcdefghij1234567890xx"),
285+
token: "gldt-",
286+
},
287+
{
288+
name: "runner authentication token",
289+
text: []byte("RUNNER_TOKEN=glrt-abcdefghij1234567890xx"),
290+
token: "glrt-",
291+
},
292+
{
293+
name: "runner registration token",
294+
text: []byte("RUNNER_TOKEN=glrtr-abcdefghij1234567890x"),
295+
token: "glrtr-",
296+
},
297+
{
298+
name: "legacy runner token",
299+
text: []byte("TOKEN=GR1348941abcdefghij1234567890"),
300+
token: "GR1348941",
301+
},
302+
}
303+
304+
for _, tt := range tests {
305+
t.Run(tt.name, func(t *testing.T) {
306+
// Use verification=true to prove that the built-in rules catch
307+
// GitLab tokens even when TruffleHog verification is active.
308+
findings, err := DetectHits(tt.text, 1, true, 60*time.Second)
309+
if err != nil {
310+
t.Fatalf("DetectHits() error = %v", err)
311+
}
312+
313+
foundByBuiltinRule := false
314+
for _, f := range findings {
315+
if strings.Contains(f.Text, tt.token) && f.Pattern.Pattern.Confidence == "high" {
316+
foundByBuiltinRule = true
317+
break
318+
}
319+
}
320+
if !foundByBuiltinRule {
321+
t.Errorf("Expected GitLab token %q to be detected by built-in rule, findings: %v", tt.token, findings)
322+
}
323+
})
324+
}
325+
}

‎pkg/scanner/rules/rules.go‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,41 @@ func InitRules(confidenceFilter []string) {
104104
func AppendPipeleekRules(rules []types.PatternElement) []types.PatternElement {
105105
customRules := []types.PatternElement{}
106106
customRules = append(customRules, types.PatternElement{Pattern: types.PatternPattern{Name: "Gitlab - Predefined Environment Variable", Regex: `(GITLAB_USER_ID|KUBECONFIG|CI_SERVER_TLS_KEY_FILE|CI_REPOSITORY_URL|CI_REGISTRY_PASSWORD|DOCKER_AUTH_CONFIG)=.*`, Confidence: "medium"}})
107+
108+
// Built-in rules for GitLab token types to ensure detection regardless of
109+
// TruffleHog verification (which only verifies against gitlab.com and
110+
// therefore misses tokens for self-hosted GitLab instances).
111+
customRules = append(customRules,
112+
// https://github.com/trufflesecurity/trufflehog/blob/main/pkg/detectors/gitlab/v2/gitlab_v2.go
113+
types.PatternElement{Pattern: types.PatternPattern{Name: "Gitlab - Personal Access Token v2", Regex: `glpat-[a-zA-Z0-9\-=_]{20,22}`, Confidence: "high"}},
114+
// https://github.com/trufflesecurity/trufflehog/blob/afd5336caad0f61da51750ffe39869974b27b0db/pkg/detectors/gitlab/v3/gitlab_v3.go#L34
115+
types.PatternElement{Pattern: types.PatternPattern{Name: "Gitlab - Personal Access Token v3", Regex: `\b(glpat-[a-zA-Z0-9\-=_]{27,300}.[0-9a-z]{2}.[a-z0-9]{9})\b`, Confidence: "high"}},
116+
// https://github.com/gitlabhq/gitlabhq/blob/master/app/models/ci/trigger.rb
117+
types.PatternElement{Pattern: types.PatternPattern{Name: "Gitlab - Pipeline Trigger Token", Regex: `glptt-[a-zA-Z0-9\-=_]{20,}`, Confidence: "high"}},
118+
// https://github.com/gitlabhq/gitlabhq/blob/master/app/models/ci/runner.rb (CREATED_RUNNER_TOKEN_PREFIX)
119+
types.PatternElement{Pattern: types.PatternPattern{Name: "Gitlab - Runner Authentication Token", Regex: `glrt-[a-zA-Z0-9\-=_]{20,}`, Confidence: "high"}},
120+
// https://github.com/gitlabhq/gitlabhq/blob/master/app/models/ci/runner.rb (REGISTRATION_RUNNER_TOKEN_PREFIX)
121+
types.PatternElement{Pattern: types.PatternPattern{Name: "Gitlab - Runner Registration Token", Regex: `glrtr-[a-zA-Z0-9\-=_]{20,}`, Confidence: "high"}},
122+
// https://github.com/gitlabhq/gitlabhq/blob/master/app/models/deploy_token.rb
123+
types.PatternElement{Pattern: types.PatternPattern{Name: "Gitlab - Deploy Token", Regex: `gldt-[a-zA-Z0-9\-=_]{20,}`, Confidence: "high"}},
124+
// https://github.com/gitlabhq/gitlabhq/blob/master/app/models/ci/build.rb
125+
types.PatternElement{Pattern: types.PatternPattern{Name: "Gitlab - CI Build Token", Regex: `glcbt-[a-zA-Z0-9\-=_]{20,}`, Confidence: "high"}},
126+
// https://github.com/gitlabhq/gitlabhq/blob/master/spec/lib/authn/tokens/oauth_application_secret_spec.rb
127+
types.PatternElement{Pattern: types.PatternPattern{Name: "Gitlab - OAuth Application Secret", Regex: `gloas-[a-zA-Z0-9\-=_]{20,}`, Confidence: "high"}},
128+
// https://docs.gitlab.com/security/token_overview/
129+
types.PatternElement{Pattern: types.PatternPattern{Name: "Gitlab - SCIM/OAuth Access Token", Regex: `glsoat-[a-zA-Z0-9\-=_]{20,}`, Confidence: "high"}},
130+
// https://docs.gitlab.com/security/token_overview/
131+
types.PatternElement{Pattern: types.PatternPattern{Name: "Gitlab - Feed Token", Regex: `glft-[a-zA-Z0-9\-=_]{20,}`, Confidence: "high"}},
132+
// https://docs.gitlab.com/security/token_overview/
133+
types.PatternElement{Pattern: types.PatternPattern{Name: "Gitlab - Incoming Mail Token", Regex: `glimt-[a-zA-Z0-9\-=_]{20,}`, Confidence: "high"}},
134+
// https://docs.gitlab.com/security/token_overview/
135+
types.PatternElement{Pattern: types.PatternPattern{Name: "Gitlab - Feature Flags Client Token", Regex: `glffct-[a-zA-Z0-9\-=_]{20,}`, Confidence: "high"}},
136+
// https://docs.gitlab.com/security/token_overview/
137+
types.PatternElement{Pattern: types.PatternPattern{Name: "Gitlab - Agent for Kubernetes Token", Regex: `glagent-[a-zA-Z0-9\-=_]{20,}`, Confidence: "high"}},
138+
// https://github.com/gitlabhq/gitlabhq/blob/master/app/models/concerns/runners_token_prefixable.rb
139+
types.PatternElement{Pattern: types.PatternPattern{Name: "Gitlab - Runner Token (Legacy)", Regex: `GR1348941[a-zA-Z0-9\-=_]{20,}`, Confidence: "high"}},
140+
)
141+
107142
return slices.Concat(rules, customRules)
108143
}
109144

‎pkg/scanner/rules/rules_test.go‎

Lines changed: 35 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,14 +22,14 @@ func TestAppendPipeleekRules(t *testing.T) {
2222
{
2323
name: "empty rules",
2424
inputRules: []types.PatternElement{},
25-
expectedCount: 1,
25+
expectedCount: 15,
2626
},
2727
{
2828
name: "with existing rules",
2929
inputRules: []types.PatternElement{
3030
{Pattern: types.PatternPattern{Name: "Test Rule", Regex: "test", Confidence: "high"}},
3131
},
32-
expectedCount: 2,
32+
expectedCount: 16,
3333
},
3434
}
3535

@@ -318,3 +318,36 @@ func TestDownloadFile_BadOutputPath(t *testing.T) {
318318
err := downloadFile(srv.URL, "/nonexistent-dir/rules.yml", client)
319319
assert.Error(t, err)
320320
}
321+
322+
func TestAppendPipeleekRules_GitLabTokenRules(t *testing.T) {
323+
result := AppendPipeleekRules([]types.PatternElement{})
324+
325+
expectedTokenRules := []string{
326+
"Gitlab - Personal Access Token v2",
327+
"Gitlab - Personal Access Token v3",
328+
"Gitlab - Pipeline Trigger Token",
329+
"Gitlab - Runner Authentication Token",
330+
"Gitlab - Runner Registration Token",
331+
"Gitlab - Deploy Token",
332+
"Gitlab - CI Build Token",
333+
"Gitlab - OAuth Application Secret",
334+
"Gitlab - SCIM/OAuth Access Token",
335+
"Gitlab - Feed Token",
336+
"Gitlab - Incoming Mail Token",
337+
"Gitlab - Feature Flags Client Token",
338+
"Gitlab - Agent for Kubernetes Token",
339+
"Gitlab - Runner Token (Legacy)",
340+
}
341+
342+
for _, expectedName := range expectedTokenRules {
343+
found := false
344+
for _, rule := range result {
345+
if rule.Pattern.Name == expectedName {
346+
found = true
347+
assert.Equal(t, "high", rule.Pattern.Confidence, "GitLab token rule %q should have high confidence", expectedName)
348+
break
349+
}
350+
}
351+
assert.True(t, found, "Expected GitLab token rule %q to be present", expectedName)
352+
}
353+
}

0 commit comments

Comments
 (0)