Skip to content

Commit

Permalink
Merge pull request #8 from vpoluyaktov/feature_Author_Search
Browse files Browse the repository at this point in the history 
Search by Creator implemented
  • Loading branch information
@vpoluyaktov
vpoluyaktov authored Dec 22, 2023
2 parents fe712c9 + be43039 commit f227e04
Show file tree
Hide file tree
Showing 17 changed files with 211 additions and 127 deletions.
29 changes: 20 additions & 9 deletions internal/config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,8 @@ var (

// Fields of this stuct should to be private but I have to make them public because yaml.Marshal/Unmarshal can't work with private fields
type Config struct {
SearchCondition string `yaml:"SearchCondition"`
DefaultAuthor string `yaml:"DefaultAuthor"`
DefaultTitle string `yaml:"DefaultTitle"`
RowsPerPage int `yaml:"RowsPerPage"`
LogFileName string `yaml:"LogFileName"`
OutputDir string `yaml:"Outputdir"`
Expand Down Expand Up @@ -72,21 +73,22 @@ func Load() {
config.LogFileName = "abb_ia.log"
config.TmpDir = "tmp"
config.CopyToOutputDir = true
config.OutputDir = "/mnt/NAS/Audiobooks/Internet Archive"
config.OutputDir = "output"
config.LogLevel = "INFO"
config.RowsPerPage = 25
config.UseMock = false
config.SaveMock = false
config.SearchCondition = ""
config.DefaultAuthor = "Old Time Radio Researchers Group"
config.DefaultTitle = "Single Episodes"
config.ConcurrentDownloaders = 5
config.ConcurrentEncoders = 5
config.ReEncodeFiles = true
config.BasePortNumber = 31000
config.BitRateKbs = 128
config.SampleRateHz = 44100
config.MaxFileSizeMb = 250
config.UploadToAudiobookshef = true
config.ScanAudiobookshef = true
config.UploadToAudiobookshef = false
config.ScanAudiobookshef = false
config.AudiobookshelfUser = "admin"
config.AudiobookshelfPassword = ""
config.AudiobookshelfLibrary = "Internet Archive"
Expand All @@ -102,6 +104,7 @@ func Load() {
"History",
"Podcast",
"Nonfiction",
"News",
"Speech",
}

Expand Down Expand Up @@ -206,12 +209,20 @@ func (c *Config) IsSaveMock() bool {
return c.SaveMock
}

func (c *Config) SetSearchCondition(s string) {
c.SearchCondition = s
func (c *Config) SetDefaultAuthor(s string) {
c.DefaultAuthor = s
}

func (c *Config) GetSearchCondition() string {
return c.SearchCondition
func (c *Config) GetDefaultAuthor() string {
return c.DefaultAuthor
}

func (c *Config) SetDefaultTitle(s string) {
c.DefaultTitle = s
}

func (c *Config) GetDefaultTitle() string {
return c.DefaultTitle
}

func (c *Config) SetConcurrentDownloaders(n int) {
Expand Down
18 changes: 9 additions & 9 deletions internal/controller/search_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import (

"abb_ia/internal/config"
"abb_ia/internal/dto"
ia_client "abb_ia/internal/ia"
"abb_ia/internal/ia"
"abb_ia/internal/logger"
"abb_ia/internal/mq"
"abb_ia/internal/utils"
Expand Down Expand Up @@ -61,39 +61,39 @@ func (c *SearchController) search(cmd *dto.SearchCommand) {
c.mq.SendMessage(mq.SearchController, mq.Footer, &dto.SetBusyIndicator{Busy: true}, false)
c.totalItemsFetched = 0
c.ia = ia_client.New(config.Instance().GetRowsPerPage(), config.Instance().IsUseMock(), config.Instance().IsSaveMock())
resp := c.ia.Search(cmd.SearchCondition, "audio")
resp := c.ia.Search(cmd.Condition.Author, cmd.Condition.Title, "audio")
if resp == nil {
logger.Error(mq.SearchController + ": Failed to perform IA search with condition: " + cmd.SearchCondition)
logger.Error(mq.SearchController + ": Failed to perform IA search with condition: " + cmd.Condition.Author + " - " + cmd.Condition.Title)
}
itemsFetched, err := c.fetchDetails(resp)
if err != nil {
logger.Error(mq.SearchController + ": Failed to fetch item details: " + err.Error())
}
c.mq.SendMessage(mq.SearchController, mq.Footer, &dto.SetBusyIndicator{Busy: false}, false)
c.mq.SendMessage(mq.SearchController, mq.Footer, &dto.UpdateStatus{Message: ""}, false)
c.mq.SendMessage(mq.SearchController, mq.SearchPage, &dto.SearchComplete{SearchCondition: cmd.SearchCondition}, false)
c.mq.SendMessage(mq.SearchController, mq.SearchPage, &dto.SearchComplete{Condition: cmd.Condition}, false)
if itemsFetched == 0 {
c.mq.SendMessage(mq.SearchController, mq.SearchPage, &dto.NothingFoundError{SearchCondition: cmd.SearchCondition}, false)
c.mq.SendMessage(mq.SearchController, mq.SearchPage, &dto.NothingFoundError{Condition: cmd.Condition}, false)
}
}

func (c *SearchController) getGetNextPage(cmd *dto.GetNextPageCommand) {
logger.Info(mq.SearchController + " received " + cmd.String())
c.mq.SendMessage(mq.SearchController, mq.Footer, &dto.UpdateStatus{Message: "Fetching Internet Archive items..."}, false)
c.mq.SendMessage(mq.SearchController, mq.Footer, &dto.SetBusyIndicator{Busy: true}, false)
resp := c.ia.GetNextPage(cmd.SearchCondition, "audio")
resp := c.ia.GetNextPage(cmd.Condition.Author, cmd.Condition.Title, "audio")
if resp == nil {
logger.Error(mq.SearchController + ": Failed to perform IA search with condition: " + cmd.SearchCondition)
logger.Error(mq.SearchController + ": Failed to perform IA search with condition: " + cmd.Condition.Author + " - " + cmd.Condition.Title)
}
itemsFetched, err := c.fetchDetails(resp)
if err != nil {
logger.Error(mq.SearchController + ": Failed to fetch item details: " + err.Error())
}
c.mq.SendMessage(mq.SearchController, mq.Footer, &dto.SetBusyIndicator{Busy: false}, false)
c.mq.SendMessage(mq.SearchController, mq.Footer, &dto.UpdateStatus{Message: ""}, false)
c.mq.SendMessage(mq.SearchController, mq.SearchPage, &dto.SearchComplete{SearchCondition: cmd.SearchCondition}, false)
c.mq.SendMessage(mq.SearchController, mq.SearchPage, &dto.SearchComplete{Condition: cmd.Condition}, false)
if itemsFetched == 0 {
c.mq.SendMessage(mq.SearchController, mq.SearchPage, &dto.LastPageMessage{SearchCondition: cmd.SearchCondition}, false)
c.mq.SendMessage(mq.SearchController, mq.SearchPage, &dto.LastPageMessage{Condition: cmd.Condition}, false)
}
}

Expand Down
10 changes: 5 additions & 5 deletions internal/controller/upload_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,12 +65,12 @@ func (c *UploadController) absScan(cmd *dto.AbsScanCommand) {
absClient := audiobookshelf.NewClient(url)
err := absClient.Login(username, password)
if err != nil {
logger.Error("Can't login to audiobookshlf server: " + err.Error())
logger.Error("Can't login to audiobookshelf server: " + err.Error())
return
}
libraries, err := absClient.GetLibraries()
if err != nil {
logger.Error("Can't get a list of libraries from audiobookshlf server: " + err.Error())
logger.Error("Can't get a list of libraries from audiobookshelf server: " + err.Error())
return
}
libraryID, err := absClient.GetLibraryId(libraries, libraryName)
Expand All @@ -80,14 +80,14 @@ func (c *UploadController) absScan(cmd *dto.AbsScanCommand) {
}
err = absClient.ScanLibrary(libraryID)
if err != nil {
logger.Error("Can't launch library scan on audiobookshlf server: " + err.Error())
logger.Error("Can't launch library scan on audiobookshelf server: " + err.Error())
return
}
if err != nil {
logger.Error("Can't launch library scan on audiobookshlf server: " + err.Error())
logger.Error("Can't launch library scan on audiobookshelf server: " + err.Error())
return
}
logger.Info("A scan launched for library " + libraryName + " on audiobookshlf server")
logger.Info("A scan launched for library " + libraryName + " on audiobookshelf server")
}
c.mq.SendMessage(mq.UploadController, mq.BuildPage, &dto.ScanComplete{Audiobook: cmd.Audiobook}, true)
}
Expand Down
25 changes: 15 additions & 10 deletions internal/dto/search.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,44 +2,49 @@ package dto

import "fmt"

type SearchCondition struct {
Author string
Title string
}

type SearchCommand struct {
SearchCondition string
Condition SearchCondition
}

func (c *SearchCommand) String() string {
return fmt.Sprintf("SearchCommand: %s", c.SearchCondition)
return fmt.Sprintf("SearchCommand: Author: %s, Title: %s", c.Condition.Author, c.Condition.Title)
}

type GetNextPageCommand struct {
SearchCondition string
Condition SearchCondition
}

func (c *GetNextPageCommand) String() string {
return fmt.Sprintf("GetNextPageCommand: %s", c.SearchCondition)
return fmt.Sprintf("GetNextPageCommand: Author: %s, Title: %s", c.Condition.Author, c.Condition.Title)
}

type SearchComplete struct {
SearchCondition string
Condition SearchCondition
}

func (c *SearchComplete) String() string {
return fmt.Sprintf("SearchComplete: %s", c.SearchCondition)
return fmt.Sprintf("SearchComplete: Author: %s, Title: %s", c.Condition.Author, c.Condition.Title)
}

type NothingFoundError struct {
SearchCondition string
Condition SearchCondition
}

func (c *NothingFoundError) String() string {
return fmt.Sprintf("NothingFoundError: %s", c.SearchCondition)
return fmt.Sprintf("NothingFoundError: Author: %s, Title: %s", c.Condition.Author, c.Condition.Title)
}

type LastPageMessage struct {
SearchCondition string
Condition SearchCondition
}

func (c *LastPageMessage) String() string {
return fmt.Sprintf("LastPageMessage: %s", c.SearchCondition)
return fmt.Sprintf("LastPageMessage: Author: %s, Title: %s", c.Condition.Author, c.Condition.Title)
}

type SearchProgress struct {
Expand Down
32 changes: 20 additions & 12 deletions internal/ia/client.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,36 +44,44 @@ func New(maxSearchRows int, useMock bool, saveMock bool) *IAClient {
return client
}

func (client *IAClient) Search(searchCondition string, mediaType string) *SearchResponse {
if strings.Contains(searchCondition, IA_BASE_URL+"/details/") {
item_id := strings.Split(searchCondition, "/")[4]
func (client *IAClient) Search(author string, title string, mediaType string) *SearchResponse {
if strings.Contains(title, IA_BASE_URL+"/details/") {
item_id := strings.Split(title, "/")[4]
return client.searchByID(item_id, mediaType)
} else {
client.page = 1
return client.searchByTitle(searchCondition, mediaType)
return client.searchByTitle(author, title, mediaType)
}
}

func (client *IAClient) GetNextPage(searchCondition string, mediaType string) *SearchResponse {
if strings.Contains(searchCondition, IA_BASE_URL+"/details/") {
func (client *IAClient) GetNextPage(author string, title string, mediaType string) *SearchResponse {
if strings.Contains(title, IA_BASE_URL+"/details/") {
return &SearchResponse{}
} else {
client.page += 1
resp := client.searchByTitle(searchCondition, mediaType)
resp := client.searchByTitle(author, title, mediaType)
return resp
}
}

func (client *IAClient) searchByTitle(title string, mediaType string) *SearchResponse {
mockFile := MOCK_DIR + "/SearchByTitle.json"
func (client *IAClient) searchByTitle(author string, title string, mediaType string) *SearchResponse {
mockFile := MOCK_DIR + "/SearchByAuthorAndTitle.json"
result := &SearchResponse{}
if client.loadMockResult {
if err := utils.LoadJson(mockFile, result); err != nil {
logger.Error("IA Client SearchByTitle() mock load error: " + err.Error())
logger.Error("IA Client SearchByAuthorAndTitle() mock load error: " + err.Error())
}
} else {
var searchURL = fmt.Sprintf(IA_BASE_URL+"/advancedsearch.php?q=title:(%s)+AND+mediatype:(%s)&output=json&rows=%d&page=%d",
url.QueryEscape(title), mediaType, client.maxSearchRows, client.page)
searchCondition := ""
if author != "" && title != "" {
searchCondition = fmt.Sprintf("creator:(%s)+AND+title:(%s)", url.QueryEscape(author), url.QueryEscape(title))
} else if author != "" {
searchCondition = fmt.Sprintf("creator:(%s)", url.QueryEscape(author))
} else if title != "" {
searchCondition = fmt.Sprintf("title:(%s)", url.QueryEscape(title))
}
var searchURL = fmt.Sprintf(IA_BASE_URL+"/advancedsearch.php?q=%s+AND+mediatype:(%s)&output=json&rows=%d&page=%d",
searchCondition, mediaType, client.maxSearchRows, client.page)
logger.Debug("IA request: " + searchURL)
_, err := client.restyClient.R().SetResult(result).Get(searchURL)
if err != nil {
Expand Down
12 changes: 10 additions & 2 deletions internal/ia/client_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,19 @@ func TestMain(m *testing.M) {
func TestSearch(t *testing.T) {
ia := ia_client.New(5, false, false)

res := ia.Search("Single Episodes", "audio") // search by title
res := ia.Search("Old Time Radio Researchers", "Single Episodes", "audio") // search by author and title
assert.NotNil(t, res)
assert.Equal(t, 5, len(res.Response.Docs))

res = ia.Search("https://archive.org/details/OTRR_Frank_Race_Singles", "audio") // search by item ID
res = ia.Search("Old Time Radio Researchers", "", "audio") // search by author only
assert.NotNil(t, res)
assert.Equal(t, 5, len(res.Response.Docs))

res = ia.Search("", "Single Episodes", "audio") // search by title only
assert.NotNil(t, res)
assert.Equal(t, 5, len(res.Response.Docs))

res = ia.Search("", "https://archive.org/details/OTRR_Frank_Race_Singles", "audio") // search by item ID
assert.NotNil(t, res)
assert.Equal(t, 1, len(res.Response.Docs))
}
Expand Down
55 changes: 37 additions & 18 deletions internal/ia/model.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ type SearchResponse struct {
NumFound int `json:"numFound"`
Start int `json:"start"`
Docs []struct {
AvgRating float64 `json:"avg_rating"`
Btih strArray `json:"btih"`
Collection []string `json:"collection"`
Creator strArray `json:"creator,omitempty"`
Date time.Time `json:"date,omitempty"`
Expand All @@ -34,6 +36,7 @@ type SearchResponse struct {
Month int `json:"month"`
OaiUpdatedate []time.Time `json:"oai_updatedate"`
Publicdate time.Time `json:"publicdate"`
Reviewdate time.Time `json:"reviewdate"`
Subject strArray `json:"subject,omitempty"`
Title string `json:"title"`
Week int `json:"week"`
Expand All @@ -52,28 +55,44 @@ type ItemDetails struct {
Server string `json:"server"`
Dir string `json:"dir"`
Metadata struct {
Identifier []string `json:"identifier"`
Creator []string `json:"creator"`
Artist []string `json:"artist"`
Date []string `json:"date"`
Description []string `json:"description"`
GUID []string `json:"guid"`
Mediatype []string `json:"mediatype"`
Rssfeed []string `json:"rssfeed"`
Scanner []string `json:"scanner"`
Sessionid []string `json:"sessionid"`
Subject []string `json:"subject"`
Title []string `json:"title"`
Uploadsoftware []string `json:"uploadsoftware"`
Collection []string `json:"collection"`
Publicdate []string `json:"publicdate"`
Addeddate []string `json:"addeddate"`
Curation []string `json:"curation"`
AccessRestrictedItem []string `json:"access-restricted-item"`
Identifier []string `json:"identifier"`
Creator []string `json:"creator"`
Artist []string `json:"artist"`
Date []string `json:"date"`
Description []string `json:"description"`
GUID []string `json:"guid"`
Mediatype []string `json:"mediatype"`
Rssfeed []string `json:"rssfeed"`
Scanner []string `json:"scanner"`
Sessionid []string `json:"sessionid"`
Subject []string `json:"subject"`
Title []string `json:"title"`
Uploadsoftware []string `json:"uploadsoftware"`
Collection []string `json:"collection"`
Licenseurl []string `json:"licenseurl"`
Notes []string `json:"notes"`
Publicdate []string `json:"publicdate"`
Addeddate []string `json:"addeddate"`
Curation []string `json:"curation"`
BackupLocation []string `json:"backup_location"`
AccessRestrictedItem []string `json:"access-restricted-item"`
ExternalMetadataUpdate []string `json:"external_metadata_update"`
Reviews struct {
Info map[string]string `json:"info"`
Reviews []struct {
ReviewBody string `json:"reviewbody"`
ReviewTitle string `json:"reviewtitle"`
Reviewer string `json:"reviewer"`
ReviewerItemname string `json:"reviewer_itemname"`
ReviewDate string `json:"reviewdate"`
Stars string `json:"stars"`
} `json:"reviews"`
} `json:"reviews"`
} `json:"metadata"`
Files map[string]struct {
Source string `json:"source"`
Format string `json:"format"`
Original string `json:"original"`
Length string `json:"length"`
Mtime string `json:"mtime"`
Size string `json:"size"`
Expand Down
2 changes: 1 addition & 1 deletion internal/mq/dispatcher.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,12 +75,12 @@ func (d *Dispatcher) SendMessage(from string, to string, dto dto.Dto, async bool
m.Async = async

if async {
d.mu.Lock()
// push message to queue
if _, ok := d.recipients[m.To]; !ok {
d.recipients[m.To] = messageQueue{list.New()}
}
// check if such message is already in queue
d.mu.Lock()
if !d.messageExists(m) {
d.recipients[m.To].messages.PushBack(m)
logger.Debug("MQ <-- async " + m.String())
Expand Down
Loading

0 comments on commit f227e04

Please sign in to comment.