feat(processor): StripComments chroma engine with legacy fallback

internal/processor/engine.go

@@ -6,6 +6,8 @@ import (
 
 	"github.com/alecthomas/chroma/v2"
 	"github.com/alecthomas/chroma/v2/lexers"
+	"github.com/carlosarraes/shush/internal/config"
+	"github.com/carlosarraes/shush/internal/types"
 )
 
 type Options struct {
@@ -61,3 +63,160 @@ func leadingWhitespace(s string) string {
 	}
 	return s
 }
+
+func StripComments(src []byte, filename string, cfg *config.Config, opts Options) ([]byte, []LineChange, error) {
+	lexer := resolveLexer(filename)
+	if lexer == nil {
+		return legacyStrip(src, filename, cfg, opts)
+	}
+	it, err := lexer.Tokenise(nil, string(src))
+	if err != nil {
+		return legacyStrip(src, filename, cfg, opts)
+	}
+	toks := it.Tokens()
+
+	var check strings.Builder
+	for _, t := range toks {
+		check.WriteString(t.Value)
+	}
+	if check.String() != string(src) {
+		return legacyStrip(src, filename, cfg, opts)
+	}
+
+	origLines := strings.Split(string(src), "\n")
+	origLineText := func(n int) string {
+		if n >= 1 && n-1 < len(origLines) {
+			return origLines[n-1]
+		}
+		return ""
+	}
+
+	var changes []LineChange
+	var out strings.Builder
+	var cur strings.Builder
+	line := 1
+	hadDrop := false
+	hadPreserved := false
+
+	flush := func() {
+		result := cur.String()
+		orig := origLineText(line)
+		action := "kept"
+		emit := true
+		if hadDrop {
+			trimmed := strings.TrimRight(result, " \t")
+			if strings.TrimSpace(trimmed) == "" {
+				action = "removed"
+				if opts.PreserveLines {
+					result = leadingWhitespace(orig)
+				} else {
+					emit = false
+					result = ""
+				}
+			} else {
+				result = trimmed
+				action = "modified"
+			}
+		} else if hadPreserved {
+			action = "preserved"
+		}
+		changes = append(changes, LineChange{Num: line, Original: orig, Result: result, Action: action})
+		if emit {
+			out.WriteString(result)
+			out.WriteString("\n")
+		}
+		cur.Reset()
+		hadDrop = false
+		hadPreserved = false
+		line++
+	}
+
+	for _, t := range toks {
+		startLine := line
+		endLine := startLine + strings.Count(t.Value, "\n")
+		drop := shouldDrop(t.Type, opts)
+		if drop && cfg != nil && cfg.ShouldPreserveComment(t.Value) {
+			drop = false
+		}
+		if drop && opts.InRange != nil {
+			for ln := startLine; ln <= endLine; ln++ {
+				if !opts.InRange(ln) {
+					drop = false
+					break
+				}
+			}
+		}
+
+		segs := strings.Split(t.Value, "\n")
+		for si, seg := range segs {
+			if drop {
+				if seg != "" {
+					hadDrop = true
+				}
+			} else {
+				cur.WriteString(seg)
+				if seg != "" && isCommentType(t.Type) {
+					hadPreserved = true
+				}
+			}
+			if si < len(segs)-1 {
+				flush()
+			}
+		}
+	}
+	if cur.Len() > 0 {
+		flush()
+	}
+
+	return []byte(out.String()), changes, nil
+}
+
+func legacyStrip(src []byte, filename string, cfg *config.Config, opts Options) ([]byte, []LineChange, error) {
+	lang, err := DetectLanguage(filename)
+	if err != nil {
+		return src, nil, err
+	}
+	p := &Processor{cli: types.CLI{Inline: opts.Inline, Block: opts.Block, PreserveLines: opts.PreserveLines}}
+	text := strings.TrimSuffix(string(src), "\n")
+	var lines []string
+	if len(src) > 0 {
+		lines = strings.Split(text, "\n")
+	}
+
+	var changes []LineChange
+	var out strings.Builder
+	for i, ln := range lines {
+		num := i + 1
+		process := opts.InRange == nil || opts.InRange(num)
+		result := ln
+		action := "kept"
+		emit := true
+		if process {
+			nl := p.removeCommentsFromLine(ln, lang, cfg)
+			if nl != ln {
+				if nl == "" {
+					action = "removed"
+					emit = false
+				} else {
+					action = "modified"
+					result = nl
+				}
+			}
+		}
+		changes = append(changes, LineChange{Num: num, Original: ln, Result: result, Action: action})
+		if emit {
+			out.WriteString(result)
+			out.WriteString("\n")
+		}
+	}
+	return []byte(out.String()), changes, nil
+}
+
+func changed(changes []LineChange) bool {
+	for _, c := range changes {
+		if c.Action == "modified" || c.Action == "removed" {
+			return true
+		}
+	}
+	return false
+}

internal/processor/engine_test.go

@@ -1,9 +1,11 @@
 package processor
 
 import (
+	"bytes"
 	"testing"
 
 	"github.com/alecthomas/chroma/v2"
+	"github.com/carlosarraes/shush/internal/config"
 )
 
 func TestResolveLexer(t *testing.T) {
@@ -56,3 +58,41 @@ func TestShouldDrop(t *testing.T) {
 		}
 	}
 }
+
+func TestStripComments(t *testing.T) {
+	def := config.Default()
+	todo := &config.Config{Preserve: []string{"TODO:"}, ContextLines: 3}
+	cases := []struct {
+		name string
+		file string
+		src  string
+		cfg  *config.Config
+		opts Options
+		want string
+	}{
+		{"multiline block collapses", "a.c", "code();\n/* a\n b */\nmore();\n", def, Options{}, "code();\nmore();\n"},
+		{"inline block both sides kept", "a.c", "x(); /* c */ y();\n", def, Options{}, "x();  y();\n"},
+		{"preproc kept comment dropped", "a.c", "#include <stdio.h>\nint x; // c\n", def, Options{}, "#include <stdio.h>\nint x;\n"},
+		{"unterminated block to eof", "a.c", "x();\n/* no end\nmore\n", def, Options{}, "x();\n"},
+		{"marker inside line comment inert", "a.c", "z(); // a /* b\nw();\n", def, Options{}, "z();\nw();\n"},
+		{"raw string preserved", "a.go", "s := `a /* b */ c`\n", def, Options{}, "s := `a /* b */ c`\n"},
+		{"string marker preserved", "a.js", "console.log(\"/* x */\"); /* r */ y();\n", def, Options{}, "console.log(\"/* x */\");  y();\n"},
+		{"preserve pattern keeps line", "a.go", "x := 1 // TODO: keep\n", todo, Options{}, "x := 1 // TODO: keep\n"},
+		{"preserve-lines blanks comment-only", "a.py", "    # c\nx = 1\n", def, Options{PreserveLines: true}, "    \nx = 1\n"},
+		{"inline flag keeps block", "a.c", "code(); /* b */ // l\n", def, Options{Inline: true}, "code(); /* b */\n"},
+		{"block flag keeps line", "a.c", "code(); /* b */ // l\n", def, Options{Block: true}, "code();  // l\n"},
+		{"straddle block kept", "a.c", "/* a\n b */ x()\n", def, Options{InRange: func(n int) bool { return n == 2 }}, "/* a\n b */ x()\n"},
+		{"range block removed when fully inside", "a.c", "x()\n/* a\n b */\ny()\n", def, Options{InRange: func(n int) bool { return n >= 2 && n <= 3 }}, "x()\ny()\n"},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			out, _, err := StripComments([]byte(c.src), c.file, c.cfg, c.opts)
+			if err != nil {
+				t.Fatalf("err: %v", err)
+			}
+			if !bytes.Equal(out, []byte(c.want)) {
+				t.Errorf("got %q want %q", out, c.want)
+			}
+		})
+	}
+}