init 1

.github/FUNDING.yml

@@ -0,0 +1,2 @@
+ko_fi: dwisiswant0
+custom: ["https://www.buymeacoffee.com/dw1", "https://paypal.me/dw1s", "https://saweria.co/dwisiswant0"]

.github/workflows/release.yaml

@@ -0,0 +1,27 @@
+name: Release
+on:
+  create:
+    tags:
+      - v*
+
+jobs: 
+  release: 
+    runs-on: ubuntu-latest
+    steps: 
+      - name: "Check out code"
+        uses: actions/checkout@v2
+        with: 
+          fetch-depth: 0
+
+      - name: "Set up Go"
+        uses: actions/setup-go@v2
+        with: 
+          go-version: 1.11
+
+      - name: "Create release on GitHub"
+        uses: goreleaser/goreleaser-action@v2
+        env: 
+          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+        with: 
+          args: "release --rm-dist"
+          version: latest

.goreleaser.yaml

@@ -0,0 +1,21 @@
+builds:
+  - binary: unew
+    main: main.go
+    goos:
+      - linux
+      - windows
+      - darwin
+    goarch:
+      - amd64
+      - 386
+      - arm
+      - arm64
+
+archives:
+  - id: tgz
+    format: tar.gz
+    replacements:
+        darwin: macOS
+    format_overrides:
+        - goos: windows
+          format: zip

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 dwisiswant0
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,69 @@
+# unew
+
+**u**_**(rl)**_**new** — A tool for append URLs, skipping duplicates & combine parameters. Inspired by [anew](https://github.com/tomnomnom/anew) & [qsreplace](https://github.com/tomnomnom/qsreplace).
+
+# Usage
+
+```bash
+▶ cat urls.txt | unew
+# or
+▶ unew urls.txt
+# or, save the results
+▶ unew urls.txt output.txt
+```
+
+## Flags
+
+Usage of `unew`:
+```
+  -combine
+      Combine parameters
+  -r string
+      Replace parameters value
+```
+
+# Install
+
+with [Go](https://golang.org/doc/install):
+
+```bash
+▶ go get -u github.com/dwisiswant0/unew
+```
+
+# Workaround
+
+If you have a list as
+
+```txt
+https://twitter.com/dwisiswant0?href=evilzone.org
+https://twitter.com/dwisiswant0
+https://twitter.com/dwisiswant0?ref=github&utm_source=github
+https://www.linkedin.com/in/dwisiswanto/
+https://www.linkedin.com/in/dwisiswanto/?originalSubdomain=id
+https://www.linkedin.com/in/dwisiswanto/?originalSubdomain=id&utm_medium=github
+```
+
+Sample workaround
+```bash
+▶ cat urls.txt | unew
+https://twitter.com/dwisiswant0?href=evilzone.org
+https://www.linkedin.com/in/dwisiswanto/
+```
+
+If the list contains multiple URLs with same path, it will save the first one and its parameters.
+
+But you can combine parameters if the same path exists by using `-combine` flag.
+
+```bash
+▶ cat urls.txt | unew -combine
+https://twitter.com/dwisiswant0?href=evilzone.org&ref=github&utm_source=github
+https://www.linkedin.com/in/dwisiswanto/?originalSubdomain=id&utm_medium=github
+```
+
+Use the `-r` flag if you want to change the value of all parameters.
+
+```bash
+▶ cat urls.txt | unew -combine -r "/etc/passwd"
+https://twitter.com/dwisiswant0?href=%2Fetc%2Fpasswd&ref=%2Fetc%2Fpasswd&utm_source=%2Fetc%2Fpasswd
+https://www.linkedin.com/in/dwisiswanto/?originalSubdomain=%2Fetc%2Fpasswd&utm_medium=%2Fetc%2Fpasswd
+```

main.go

@@ -0,0 +1,136 @@
+package main
+
+import (
+	"bufio"
+	"flag"
+	"fmt"
+	"net/url"
+	"os"
+	"strings"
+)
+
+var combine bool
+var outfile *os.File
+var sc *bufio.Scanner
+var replace, outtext, fr, fs string
+
+func init() {
+	flag.StringVar(&replace, "r", "", "Replace parameters value")
+	flag.BoolVar(&combine, "combine", false, "Combine parameters")
+	flag.Parse()
+
+	fr := flag.Arg(0)
+
+	if isStdin() {
+		sc = bufio.NewScanner(os.Stdin)
+	} else if fr != "" {
+		r, err := os.Open(fr)
+		if err == nil {
+			sc = bufio.NewScanner(r)
+		}
+	} else {
+		os.Exit(1)
+	}
+}
+
+func main() {
+	urls := make(map[string]string)
+
+	fs := flag.Arg(1)
+	if fs != "" {
+		outfile, _ = os.OpenFile(fs, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0644)
+		defer outfile.Close()
+	}
+
+	for sc.Scan() {
+		u, err := url.ParseRequestURI(sc.Text())
+		if err != nil {
+			continue
+		}
+
+		b := fmt.Sprintf("%s://%s%s", u.Scheme, u.Host, u.Path)
+
+		if _, d := urls[b]; d {
+			if combine {
+				q := []string{urls[b], u.RawQuery}
+				u.RawQuery = remDup(strings.Join(q, "&"))
+			} else {
+				continue
+			}
+		} else {
+			urls[b] = u.RawQuery
+		}
+
+		if replace != "" {
+			u.RawQuery = qsReplace(u.Query(), replace)
+		}
+
+		if !combine {
+			outtext = fmt.Sprintf("%s%s\n", b, qMark(u.RawQuery))
+			fmt.Printf("%s", outtext)
+			if fs != "" {
+				fmt.Fprintf(outfile, "%s", outtext)
+			}
+
+			continue
+		}
+
+		urls[b] = u.RawQuery
+	}
+
+	if !combine {
+		return
+	}
+
+	for k, v := range urls {
+		outtext = fmt.Sprintf("%s%s\n", k, qMark(v))
+		fmt.Printf("%s", outtext)
+		if fs != "" {
+			fmt.Fprintf(outfile, "%s", outtext)
+		}
+	}
+}
+
+func isStdin() bool {
+	f, e := os.Stdin.Stat()
+	if e != nil {
+		return false
+	}
+
+	if f.Mode()&os.ModeNamedPipe == 0 {
+		return false
+	}
+
+	return true
+}
+
+func qMark(q string) string {
+	if q == "" {
+		return ""
+	}
+
+	return "?" + q
+}
+
+func remDup(q string) string {
+	qs := url.Values{}
+	ps, _ := url.ParseQuery(q)
+
+	for p, v := range ps {
+		for range v {
+			v = v[:1:1]
+			qs.Set(p, v[0])
+		}
+	}
+
+	return qs.Encode()
+}
+
+func qsReplace(q url.Values, r string) string {
+	qs := url.Values{}
+	for p := range q {
+		qs.Set(p, r)
+	}
+
+	return qs.Encode()
+}