Skip to content

Commit

Permalink
move to cron
Browse files Browse the repository at this point in the history
  • Loading branch information
m1yon committed May 3, 2024
1 parent 512f8dd commit b78779e
Show file tree
Hide file tree
Showing 3 changed files with 23 additions and 3 deletions.
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ require (
github.com/prometheus/common v0.48.0 // indirect
github.com/prometheus/procfs v0.12.0 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
github.com/robfig/cron v1.2.0 // indirect
github.com/sethvargo/go-retry v0.2.4 // indirect
github.com/ysmood/fetchup v0.2.3 // indirect
github.com/ysmood/goob v0.4.0 // indirect
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,8 @@ github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k
github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/robfig/cron v1.2.0 h1:ZjScXvvxeQ63Dbyxy76Fj3AT3Ut0aKsyd2/tl3DTMuQ=
github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k=
github.com/sethvargo/go-retry v0.2.4 h1:T+jHEQy/zKJf5s95UkguisicE0zuF9y7+/vgz08Ocec=
github.com/sethvargo/go-retry v0.2.4/go.mod h1:1afjQuvh7s4gflMObvjLPaWgluLLyhA1wmVZ6KLpICw=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
Expand Down
23 changes: 20 additions & 3 deletions internal/linkedincrawler/linkedincrawler.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"github.com/go-rod/stealth"
"github.com/lmittmann/tint"
"github.com/m1yon/jobsummoner/internal/database"
"github.com/robfig/cron"
"modernc.org/sqlite"
sqlite3 "modernc.org/sqlite/lib"
)
Expand All @@ -44,13 +45,29 @@ func ScrapeLoop(db *sql.DB) {
},
}

for {
location, err := time.LoadLocation("America/Denver")

if err != nil {
slog.Error("failed creating cron location", tint.Err(err))
}

c := cron.NewWithLocation(location)

for _, options := range scrapes {
c.AddFunc("*/30 7-22 * * *", func() {
scrape(db, options)
})
}

scrapeOnStart := os.Getenv("SCRAPE_ON_START")

if scrapeOnStart == "true" {
for _, options := range scrapes {
scrape(db, options)
}

time.Sleep(60 * time.Minute)
}

c.Start()
}

type scrapeOptions struct {
Expand Down

0 comments on commit b78779e

Please sign in to comment.