From 0bfae00480e48df0a2aa857af4153ba687898644 Mon Sep 17 00:00:00 2001 From: Wojciech Bednarzak Date: Wed, 10 Mar 2021 22:23:47 +0000 Subject: [PATCH] Add database interface for shows and episode This abstracts getting the data from the database. In the commit there are 2 implementations: consul and sql. --- internal/database/consul/database.go | 30 ++++- internal/database/consul/database_test.go | 38 ++++++ internal/database/consul/shows.go | 81 ++++++++++++ internal/database/consul/shows_test.go | 71 ++++++++++ internal/database/consul/users.go | 4 +- .../consul/{consul_test.go => users_test.go} | 22 +--- internal/database/database.go | 11 +- internal/database/sql/shows.go | 123 ++++++++++++++++++ internal/database/sql/sql_test.go | 2 +- internal/frontend/show.go | 2 +- internal/types/show/show.go | 20 +++ trackable/show/collector.go | 3 +- trackable/show/show.go | 39 +++++- 13 files changed, 414 insertions(+), 32 deletions(-) create mode 100644 internal/database/consul/database_test.go create mode 100644 internal/database/consul/shows.go create mode 100644 internal/database/consul/shows_test.go rename internal/database/consul/{consul_test.go => users_test.go} (68%) create mode 100644 internal/database/sql/shows.go create mode 100644 internal/types/show/show.go diff --git a/internal/database/consul/database.go b/internal/database/consul/database.go index f7ea334..2107c67 100644 --- a/internal/database/consul/database.go +++ b/internal/database/consul/database.go @@ -6,14 +6,14 @@ import ( "encoding/json" "fmt" "path" + "strings" "github.com/hashicorp/consul/api" "tracker/internal/database" ) -// UserDatabase implements database.UserDatabase and can be used -// to insert user data. +// Database contains methods for getting the data about shows and episodes type Database struct { kv KV @@ -39,7 +39,7 @@ func NewDatabase(prefix string, opts ...Option) (*Database, error) { } // get value from the database -func (db *Database) get(ctx context.Context, key string, value interface{}) error { +func (db *Database) get(ctx context.Context, key string, value any) error { opt := &api.QueryOptions{} opt = opt.WithContext(ctx) @@ -60,7 +60,7 @@ func (db *Database) get(ctx context.Context, key string, value interface{}) erro return nil } -func (db *Database) put(ctx context.Context, key string, value interface{}) error { +func (db *Database) put(ctx context.Context, key string, value any) error { opt := &api.WriteOptions{} opt = opt.WithContext(ctx) @@ -79,6 +79,27 @@ func (db *Database) put(ctx context.Context, key string, value interface{}) erro } // Option allows to set options for the Consul database.type Option func(*Database) +func (db *Database) list(ctx context.Context, prefix string) (map[string][]byte, error) { + opt := &api.QueryOptions{} + opt = opt.WithContext(ctx) + + p := path.Join(db.prefix, prefix) + + kvs, _, err := db.kv.List(p, opt) + if err != nil { + return nil, fmt.Errorf("unable to list %s: %w", p, err) + } + + m := make(map[string][]byte, len(kvs)) + for _, kv := range kvs { + // Remove all the prefixes from the keys. + m[strings.TrimPrefix(kv.Key, p+"/")] = kv.Value + } + + return m, nil +} + +// Option allows to set options for the Consul database. type Option func(*Database) func KVClient(kv KV) Option { @@ -91,4 +112,5 @@ func KVClient(kv KV) Option { type KV interface { Put(p *api.KVPair, q *api.WriteOptions) (*api.WriteMeta, error) Get(key string, q *api.QueryOptions) (*api.KVPair, *api.QueryMeta, error) + List(prefix string, q *api.QueryOptions) (api.KVPairs, *api.QueryMeta, error) } diff --git a/internal/database/consul/database_test.go b/internal/database/consul/database_test.go new file mode 100644 index 0000000..400ae10 --- /dev/null +++ b/internal/database/consul/database_test.go @@ -0,0 +1,38 @@ +package consul + +import ( + "errors" + "strings" + + "github.com/hashicorp/consul/api" +) + +var errNotFound = errors.New("kv: not found") + +type testKV struct { + m map[string][]byte +} + +func (kv *testKV) Get(key string, _ *api.QueryOptions) (*api.KVPair, *api.QueryMeta, error) { + if v, exists := kv.m[key]; exists { + return &api.KVPair{Key: key, Value: v}, nil, nil + } else { + return nil, nil, errNotFound + } +} + +func (kv *testKV) Put(pair *api.KVPair, _ *api.WriteOptions) (*api.WriteMeta, error) { + kv.m[pair.Key] = pair.Value + return nil, nil +} + +func (kv *testKV) List(prefix string, _ *api.QueryOptions) (api.KVPairs, *api.QueryMeta, error) { + var pairs []*api.KVPair + for key, value := range kv.m { + if strings.HasPrefix(key, prefix) { + pairs = append(pairs, &api.KVPair{Key: key, Value: value}) + } + } + + return api.KVPairs(pairs), nil, nil +} diff --git a/internal/database/consul/shows.go b/internal/database/consul/shows.go new file mode 100644 index 0000000..734b00f --- /dev/null +++ b/internal/database/consul/shows.go @@ -0,0 +1,81 @@ +package consul + +import ( + "context" + "encoding/json" + "fmt" + "path" + + "tracker/internal/types/show" +) + +type ShowsDatabase struct { + db *Database + prefix string +} + +func (db *Database) Shows() *ShowsDatabase { + return &ShowsDatabase{ + db: db, + prefix: "shows", + } +} + +func (db *ShowsDatabase) List(ctx context.Context) ([]*show.Show, error) { + showIDs, err := db.list(ctx, "list") + if err != nil { + return nil, fmt.Errorf("unable to get show IDs") + } + + // TODO: Speed up by running these in parallel + var shows []*show.Show + for showID := range showIDs { + show, err := db.showDetails(ctx, showID) + if err != nil { + return nil, fmt.Errorf("unable to get show %s: %w", showID, err) + } + shows = append(shows, show) + } + + return shows, nil +} + +func (db *ShowsDatabase) Details(ctx context.Context, id string) (*show.Show, error) { + // TODO: Run getting show details and show episodes in parallel + s, err := db.showDetails(ctx, id) + if err != nil { + return nil, fmt.Errorf("unable to get details about %s: %w", id, err) + } + + episodes, err := db.list(ctx, path.Join("episodes", id)) + if err != nil { + return nil, fmt.Errorf("unable to get episodes for %s: %w", id, err) + } + + for _, ep := range episodes { + var episode show.Episode + if err := json.Unmarshal(ep, &episode); err != nil { + return nil, fmt.Errorf("unable to parse episode: %w", err) + } + s.Episodes = append(s.Episodes, &episode) + } + + return s, nil +} + +func (db *ShowsDatabase) showDetails(ctx context.Context, id string) (*show.Show, error) { + var show show.Show + if err := db.get(ctx, path.Join("list", id), &show); err != nil { + return nil, err + } + + return &show, nil +} + +func (db *ShowsDatabase) get(ctx context.Context, key string, value any) error { + return db.db.get(ctx, path.Join(db.prefix, key), value) +} + +func (db *ShowsDatabase) list(ctx context.Context, prefix string) (map[string][]byte, error) { + return db.db.list(ctx, path.Join(db.prefix, prefix)) +} diff --git a/internal/database/consul/shows_test.go b/internal/database/consul/shows_test.go new file mode 100644 index 0000000..7d673ba --- /dev/null +++ b/internal/database/consul/shows_test.go @@ -0,0 +1,71 @@ +package consul + +import ( + "context" + "sort" + "testing" + "tracker/internal/types/show" + + "github.com/go-test/deep" +) + +var testData = map[string][]byte{ + "tracker/shows/list/westworld": []byte(`{"id": 1, "name": "Westworld"}`), + "tracker/shows/episodes/westworld_s01e01": []byte(`{"title":"The Original", "season": 1, "episode": 1}`), + + "tracker/shows/list/expanse": []byte(`{"id": 2, "name": "The Expanse"}`), +} + +func TestList(t *testing.T) { + kv := &testKV{m: testData} + db, err := NewDatabase("tracker", KVClient(kv)) + if err != nil { + t.Fatalf("unable to setup database: %v", err) + } + + showsDB := db.Shows() + + want := []*show.Show{ + {ID: 2, Name: "The Expanse"}, + {ID: 1, Name: "Westworld"}, + } + + got, err := showsDB.List(context.Background()) + if err != nil { + t.Fatalf("unexpected error listing shows: %v", err) + } + + sort.Sort(show.ByName(got)) + + if diff := deep.Equal(got, want); diff != nil { + t.Fatalf("List() = %v, want %v, diff = %v", got, want, diff) + } +} + +func TestDetails(t *testing.T) { + kv := &testKV{m: testData} + db, err := NewDatabase("tracker", KVClient(kv)) + if err != nil { + t.Fatalf("unable to setup database: %v", err) + } + + showsDB := db.Shows() + + want := &show.Show{ + ID: 1, + Name: "Westworld", + Episodes: []*show.Episode{ + {Title: "The Original", Season: 1, Episode: 1}, + }, + } + + got, err := showsDB.Details(context.Background(), "westworld") + if err != nil { + t.Fatalf("unexpected error getting details: %v", err) + } + + if diff := deep.Equal(got, want); diff != nil { + t.Fatalf("Details() = %v, want %v, diff = %v", got, want, diff) + } + +} diff --git a/internal/database/consul/users.go b/internal/database/consul/users.go index 5945e52..bdaca38 100644 --- a/internal/database/consul/users.go +++ b/internal/database/consul/users.go @@ -39,10 +39,10 @@ func (db *UsersDatabase) Details(ctx context.Context, email string) (*user.User, return &u, nil } -func (db *UsersDatabase) get(ctx context.Context, key string, value interface{}) error { +func (db *UsersDatabase) get(ctx context.Context, key string, value any) error { return db.db.get(ctx, path.Join(db.prefix, key), value) } -func (db *UsersDatabase) put(ctx context.Context, key string, value interface{}) error { +func (db *UsersDatabase) put(ctx context.Context, key string, value any) error { return db.db.put(ctx, path.Join(db.prefix, key), value) } diff --git a/internal/database/consul/consul_test.go b/internal/database/consul/users_test.go similarity index 68% rename from internal/database/consul/consul_test.go rename to internal/database/consul/users_test.go index b44afbd..a61391c 100644 --- a/internal/database/consul/consul_test.go +++ b/internal/database/consul/users_test.go @@ -9,13 +9,10 @@ import ( "tracker/internal/types/user" "github.com/go-test/deep" - "github.com/hashicorp/consul/api" ) -var errNotFound = errors.New("kv: not found") - func TestImplements(t *testing.T) { - var i interface{} = &UsersDatabase{} + var i any = &UsersDatabase{} if _, ok := i.(database.UsersDatabase); !ok { t.Errorf("UserDatabase does not implement database.UserDatabase") @@ -56,20 +53,3 @@ func TestE2E(t *testing.T) { t.Fatalf("Get() = %v, got %v, diff = %v", got, want, diff) } } - -type testKV struct { - m map[string][]byte -} - -func (kv *testKV) Put(pair *api.KVPair, _ *api.WriteOptions) (*api.WriteMeta, error) { - kv.m[pair.Key] = pair.Value - return nil, nil -} - -func (kv *testKV) Get(key string, _ *api.QueryOptions) (*api.KVPair, *api.QueryMeta, error) { - if v, exists := kv.m[key]; exists { - return &api.KVPair{Key: key, Value: v}, nil, nil - } else { - return nil, nil, errNotFound - } -} diff --git a/internal/database/database.go b/internal/database/database.go index fece669..74b3130 100644 --- a/internal/database/database.go +++ b/internal/database/database.go @@ -3,6 +3,7 @@ package database import ( "context" + "tracker/internal/types/show" "tracker/internal/types/user" ) @@ -16,9 +17,10 @@ const ( ErrNotFound = Error("not found") ) -// Database +// Database is the shared abstraction over all databases type Database interface { Users() UsersDatabase + Shows() ShowsDatabase } // UserDatabase abstracts the user interaction with the database. @@ -28,3 +30,10 @@ type UsersDatabase interface { // Details the user based on the email address. Details(ctx context.Context, email string) (*user.User, error) } + +type ShowsDatabase interface { + // List returns the list of shows. The episode list is empty in each show. + List(context.Context) ([]*show.Show, error) + // Details gives details about a show by a show ID, including the episodes. + Details(context.Context, int) (*show.Show, error) +} diff --git a/internal/database/sql/shows.go b/internal/database/sql/shows.go new file mode 100644 index 0000000..67d6508 --- /dev/null +++ b/internal/database/sql/shows.go @@ -0,0 +1,123 @@ +package sql + +import ( + "context" + "database/sql" + "fmt" + + "tracker/internal/types/show" +) + +type ShowsDatabase struct { + db *Database + + listStmt *sql.Stmt + detailsStmt *sql.Stmt + episodesStmt *sql.Stmt +} + +func (db *Database) Shows() *ShowsDatabase { + listStmt, err := db.db.Prepare(listQuery) + if err != nil { + // TODO: Is there a cleaner way of doing this? Is this OK? + panic(fmt.Sprintf("unable to prepare list statement: %v", err)) + } + + detailsStmt, err := db.db.Prepare(detailsQuery) + if err != nil { + // TODO: Is there a cleaner way of doing this? Is this OK? + panic(fmt.Sprintf("unable to prepare details statement: %v", err)) + } + + episodesStmt, err := db.db.Prepare(episodesQuery) + if err != nil { + // TODO: Is there a cleaner way of doing this? Is this OK? + panic(fmt.Sprintf("unable to prepare episodes statement: %v", err)) + } + + return &ShowsDatabase{ + db: db, + listStmt: listStmt, + detailsStmt: detailsStmt, + episodesStmt: episodesStmt, + } +} + +func (db *ShowsDatabase) List(ctx context.Context) ([]*show.Show, error) { + rows, err := db.listStmt.QueryContext(ctx) + if err != nil { + return nil, fmt.Errorf("unable to list shows: %w", err) + } + + var shows []*show.Show + for rows.Next() { + show := &show.Show{} + if err := rows.Scan( + &show.ID, + &show.Name, + &show.WikipediaURL, + &show.TrailerURL, + &show.Finished, + ); err != nil { + return nil, fmt.Errorf("unable to get show details: %w", err) + } + shows = append(shows, show) + } + + return shows, nil +} + +func (db *ShowsDatabase) Details(ctx context.Context, id string) (*show.Show, error) { + s := &show.Show{} + if err := db.detailsStmt.QueryRowContext(ctx, id).Scan( + &s.ID, + &s.Name, + &s.WikipediaURL, + &s.TrailerURL, + &s.Finished, + ); err != nil { + return nil, fmt.Errorf("unable to get show details: %w", err) + } + + rows, err := db.episodesStmt.QueryContext(ctx, s.ID) + if err != nil { + return nil, fmt.Errorf("unable to show episodes: %w", err) + } + + for rows.Next() { + episode := &show.Episode{} + if err := rows.Scan( + &episode.Title, + &episode.Season, + &episode.Episode, + &episode.ReleaseDate, + ); err != nil { + return nil, fmt.Errorf("unable to get episode: %w", err) + } + s.Episodes = append(s.Episodes, episode) + } + + return s, nil +} + +var listQuery = ` +SELECT + id, + title, + wikipedia, + trailer, + finished, +FROM shows +` + +var detailsQuery = listQuery + " WHERE id=? LIMIT 1" + +var episodesQuery = ` +SELECT + title, + season, + episode, + release_date +FROM episodes +WHERE show_id=? +` diff --git a/internal/database/sql/sql_test.go b/internal/database/sql/sql_test.go index 10943ca..6b3c9a5 100644 --- a/internal/database/sql/sql_test.go +++ b/internal/database/sql/sql_test.go @@ -6,7 +6,7 @@ import ( ) func TestImplements(t *testing.T) { - var i interface{} = &UsersDatabase{} + var i any = &UsersDatabase{} if _, ok := i.(database.UsersDatabase); !ok { t.Errorf("UserDatabase doesn't implement database.UserDatabase") diff --git a/internal/frontend/show.go b/internal/frontend/show.go index cd06673..8aa0432 100644 --- a/internal/frontend/show.go +++ b/internal/frontend/show.go @@ -228,7 +228,7 @@ func (f *ShowFrontend) addShowRequest(w http.ResponseWriter, r *http.Request) { // get the given URL and unmarshal data into res. The url specified must be // prefixed with a / -func (f *ShowFrontend) get(ctx context.Context, url string, v interface{}) error { +func (f *ShowFrontend) get(ctx context.Context, url string, v any) error { req, err := http.NewRequestWithContext( ctx, http.MethodGet, diff --git a/internal/types/show/show.go b/internal/types/show/show.go new file mode 100644 index 0000000..a4046b0 --- /dev/null +++ b/internal/types/show/show.go @@ -0,0 +1,20 @@ +package show + +import ( + "strings" + "tracker/trackable/show" +) + +// TODO: Move this to a concrete implementation in the future +type Show = show.Show + +type Episode = show.Episode + +// ByName allows to sort the shows alphabetically by show name. +type ByName []*Show + +func (ss ByName) Len() int { return len(ss) } +func (ss ByName) Swap(i, j int) { ss[i], ss[j] = ss[j], ss[i] } +func (ss ByName) Less(i, j int) bool { + return strings.ToLower(ss[i].Name) < strings.ToLower(ss[j].Name) +} diff --git a/trackable/show/collector.go b/trackable/show/collector.go index 65c6ff3..f41124b 100644 --- a/trackable/show/collector.go +++ b/trackable/show/collector.go @@ -146,10 +146,11 @@ func (s *Show) parseEpisodeTable(table *scrape.Tag, season int, previousDate tim // Get release date text := parseString(column.Text()) if timeutil.HasMonth(text) { - episode.ReleaseDate, err = timeutil.Parse(text) + rd, err := timeutil.Parse(text) if err != nil { fmt.Printf("Unable to convert %s to a date object: %v\n", text, err) } + episode.ReleaseDate = time.Time(rd) continue } diff --git a/trackable/show/show.go b/trackable/show/show.go index 453ae2e..8b4fe26 100644 --- a/trackable/show/show.go +++ b/trackable/show/show.go @@ -2,11 +2,13 @@ package show import ( "database/sql" + "encoding/json" "fmt" "sort" "time" "tracker/database" + "tracker/internal/timeutil" _ "github.com/go-sql-driver/mysql" ) @@ -36,6 +38,42 @@ type Episode struct { ReleaseDate time.Time } +type episodeJSON struct { + Title string `json:"title"` + Season int `json:"season"` + Episode int `json:"episode"` + ReleaseDateJSON timeutil.JSONTime `json:"release_date"` +} + +// MarshalJSON add the proper JSON encoding to the release_date +func (e *Episode) MarshalJSON() ([]byte, error) { + ej := episodeJSON{ + Title: e.Title, + Season: e.Season, + Episode: e.Episode, + ReleaseDateJSON: timeutil.JSONTime(e.ReleaseDate), + } + + return json.Marshal(ej) +} + +// UnmarshalJSON converts back to episode with release_date correctly parsed. +func (e *Episode) UnmarshalJSON(b []byte) error { + ej := new(episodeJSON) + if err := json.Unmarshal(b, ej); err != nil { + return err + } + + *e = Episode{ + Title: ej.Title, + Season: ej.Season, + Episode: ej.Episode, + ReleaseDate: time.Time(ej.ReleaseDateJSON), + } + + return nil +} + func (s *Show) Write() error { db, err := database.Open("tracker") if err != nil { @@ -115,7 +153,6 @@ func (s *Show) String() string { return fmt.Sprintf("%-2d - %-30s - %3d Episodes, WikipediaURL='%s'\n%s", s.ID, s.Name, len(s.Episodes), s.WikipediaURL, episodeString) - } func (s *Episode) String() string {