From 98273e164d4c5f3c7a1f601558ab4fd21df4f61b Mon Sep 17 00:00:00 2001 From: Riccardo Date: Sat, 16 Jul 2022 16:37:07 +0200 Subject: [PATCH] Added code for AWS Lambda --- db/db.go | 5 ++-- go.mod | 4 +-- go.sum | 10 +++---- handler/handler.go | 61 +++++++++++++++++++++++++++++++++++++++++ main.go | 68 ++-------------------------------------------- params/params.go | 28 ------------------- 6 files changed, 74 insertions(+), 102 deletions(-) create mode 100644 handler/handler.go delete mode 100644 params/params.go diff --git a/db/db.go b/db/db.go index 7f20745..f29a749 100644 --- a/db/db.go +++ b/db/db.go @@ -15,7 +15,6 @@ import ( type Table struct { DynamoDbClient *dynamodb.Client TableName string - IndexName string } type News struct { @@ -139,7 +138,9 @@ func ReadTodayNews(basics Table, timeStart int, timeEnd int) ([]News, error) { } } - fmt.Println("MARSH", len(response.Items), news[0]) + if len(response.Items) > 0 { + fmt.Println("MARSH", len(response.Items), news[0]) + } return news, err } diff --git a/go.mod b/go.mod index 322bd26..d5bffd2 100644 --- a/go.mod +++ b/go.mod @@ -3,10 +3,10 @@ module hackernewsletter go 1.18 require ( + github.com/aws/aws-lambda-go v1.32.1 github.com/aws/aws-sdk-go-v2 v1.16.7 github.com/aws/aws-sdk-go-v2/config v1.15.14 github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.9.6 - github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression v1.4.12 github.com/aws/aws-sdk-go-v2/service/dynamodb v1.15.9 github.com/spf13/viper v1.12.0 ) @@ -40,5 +40,5 @@ require ( golang.org/x/text v0.3.7 // indirect gopkg.in/ini.v1 v1.66.4 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index a437c8d..7cbeb0d 100644 --- a/go.sum +++ b/go.sum @@ -38,6 +38,8 @@ cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3f dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/aws/aws-lambda-go v1.32.1 h1:ls0FU8Mt7ayJszb945zFkUfzxhkQTli8mpJstVcDtCY= +github.com/aws/aws-lambda-go v1.32.1/go.mod h1:jwFe2KmMsHmffA1X2R09hH6lFzJQxzI8qK17ewzbQMM= github.com/aws/aws-sdk-go-v2 v1.16.7 h1:zfBwXus3u14OszRxGcqCDS4MfMCv10e8SMJ2r8Xm0Ns= github.com/aws/aws-sdk-go-v2 v1.16.7/go.mod h1:6CpKuLXg2w7If3ABZCl/qZ6rEgwtjZTn4eAf4RcEyuw= github.com/aws/aws-sdk-go-v2/config v1.15.14 h1:+BqpqlydTq4c2et9Daury7gE+o67P4lbk7eybiCBNc4= @@ -46,8 +48,6 @@ github.com/aws/aws-sdk-go-v2/credentials v1.12.9 h1:DloAJr0/jbvm0iVRFDFh8GlWxrOd github.com/aws/aws-sdk-go-v2/credentials v1.12.9/go.mod h1:2Vavxl1qqQXJ8MUcQZTsIEW8cwenFCWYXtLRPba3L/o= github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.9.6 h1:vlEfSyZ2pZjOZe7zsPIAFem17w2HeeFULk7TPVWoDR4= github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.9.6/go.mod h1:+/KXTIzLmrjdlQVgiE14/jhy9GyDZnmMGQoykod99Lw= -github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression v1.4.12 h1:sH0SffGPiNpvYCCfEF0dN0K9OC72KXBjW4HmiFvMVf0= -github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression v1.4.12/go.mod h1:0vvQ0FQRjyNB8EIkRdwT9tduJbkUdh00SnmuKnZRYLA= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.8 h1:VfBdn2AxwMbFyJN/lF/xuT3SakomJ86PZu3rCxb5K0s= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.8/go.mod h1:oL1Q3KuCq1D4NykQnIvtRiBGLUXhcpY5pl6QZB2XEPU= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.14 h1:2C0pYHcUBmdzPj+EKNC4qj97oK6yjrUhc1KoSodglvk= @@ -200,8 +200,8 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.2 h1:4jaiDzPyXQvSd7D0EjG45355tLlV3VOECpq10pLC+8s= github.com/subosito/gotenv v1.3.0 h1:mjC+YW8QpAdXibNi+vNWgzmgBH4+5l5dCXv8cNysBLI= github.com/subosito/gotenv v1.3.0/go.mod h1:YzJjq/33h7nrwdY+iHMhEOEEbW0ovIz0tB6t6PwAXzs= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -503,8 +503,8 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0 h1:hjy8E9ON/egN1tAYqKb61G10WtihqetD4sz2H+8nIeA= -gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/handler/handler.go b/handler/handler.go new file mode 100644 index 0000000..5a396f8 --- /dev/null +++ b/handler/handler.go @@ -0,0 +1,61 @@ +package handler + +import ( + "context" + "fmt" + "hackernewsletter/db" + "hackernewsletter/hackernews" + "time" + + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/dynamodb" +) + +func Handler(ctx context.Context) (string, error) { + table := "news_table" + + cfg, err := config.LoadDefaultConfig(context.TODO()) + if err != nil { + panic("Unable to load SDK config, " + err.Error()) + } + + client := dynamodb.NewFromConfig(cfg) + + input := &dynamodb.DescribeTableInput{ + TableName: &table, + } + + newsTable := db.Table{client, table} + + resp, err := db.GetTableInfo(context.TODO(), client, input) + if err != nil { + println(("Table not found. Creating it...")) + _, new_err := db.CreateTable(newsTable) + if new_err != nil { + panic("Failed creating table " + table) + } + + resp, err = db.GetTableInfo(context.TODO(), client, input) + } + + fmt.Printf("Table %v has %v elements\n", table, resp.Table.ItemCount) + + newsIds := hackernews.GetTopNewsIds("https://hacker-news.firebaseio.com/v0/topstories.json") + + var newsBatch []db.News + + for i := 0; i < 10; i++ { + myNews := hackernews.GetNewsById(newsIds[i], "https://hacker-news.firebaseio.com/v0/item/{ID}.json") + newsBatch = append(newsBatch, myNews) + } + + db.AddNewsBatch(newsTable, newsBatch, 25) + + timeEnd := time.Date(time.Now().Year(), time.Now().Month(), time.Now().Day(), 0, 0, 0, 0, time.UTC) + timeStart := timeEnd.Add(-time.Hour * 24) + + todayNews, _ := db.ReadTodayNews(newsTable, int(timeStart.Unix()), int(timeEnd.Unix())) + fmt.Print("NEWS", todayNews) + + return "Completed.", err +} diff --git a/main.go b/main.go index 9c14f47..29debf1 100644 --- a/main.go +++ b/main.go @@ -1,73 +1,11 @@ package main import ( - "context" - "flag" - "fmt" - "hackernewsletter/db" - "hackernewsletter/hackernews" - "hackernewsletter/params" - "log" - "time" + "hackernewsletter/handler" - "github.com/aws/aws-sdk-go-v2/config" - "github.com/aws/aws-sdk-go-v2/service/dynamodb" + "github.com/aws/aws-lambda-go/lambda" ) func main() { - conf, err := params.LoadConfig(".") - if err != nil { - log.Fatal("Cannot load config:", err) - } - - table := flag.String("t", "", "The name of the table") - flag.Parse() - - if *table == "" { - fmt.Println("Table name not specified. Using default name.") - table = &conf.TableName - } - - cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithRegion(conf.AwsRegion)) - if err != nil { - panic("Unable to load SDK config, " + err.Error()) - } - - client := dynamodb.NewFromConfig(cfg) - - input := &dynamodb.DescribeTableInput{ - TableName: table, - } - - newsTable := db.Table{client, *table, conf.TableIndex} - - resp, err := db.GetTableInfo(context.TODO(), client, input) - if err != nil { - println(("Table not found. Creating it...")) - _, new_err := db.CreateTable(newsTable) - if new_err != nil { - panic("Failed creating table " + *table) - } - - resp, err = db.GetTableInfo(context.TODO(), client, input) - } - - fmt.Printf("Table %v has %v elements\n", *table, resp.Table.ItemCount) - - newsIds := hackernews.GetTopNewsIds(conf.TopNews) - - var newsBatch []db.News - - for i := 0; i < 10; i++ { - myNews := hackernews.GetNewsById(newsIds[i], conf.SingleNews) - newsBatch = append(newsBatch, myNews) - } - - db.AddNewsBatch(newsTable, newsBatch, conf.BatchSize) - - timeEnd := time.Date(time.Now().Year(), time.Now().Month(), time.Now().Day(), 0, 0, 0, 0, time.UTC) - timeStart := timeEnd.Add(-time.Hour * 24) - - todayNews, _ := db.ReadTodayNews(newsTable, int(timeStart.Unix()), int(timeEnd.Unix())) - println("NEWS", todayNews) + lambda.Start(handler.Handler) } diff --git a/params/params.go b/params/params.go deleted file mode 100644 index e0d4c05..0000000 --- a/params/params.go +++ /dev/null @@ -1,28 +0,0 @@ -package params - -import "github.com/spf13/viper" - -type Config struct { - TableName string `mapstructure:"TABLE_NAME"` - TableIndex string `mapstructure:"GLOBAL_SECONDARY_INDEX"` - AwsRegion string `mapstructure:"AWS_REGION"` - TopNews string `mapstructure:"TOP_NEWS"` - SingleNews string `mapstructure:"SINGLE_NEWS"` - BatchSize int `mapstructure:"SERVER_ADDRESS"` -} - -func LoadConfig(path string) (config Config, err error) { - viper.AddConfigPath(path) - viper.SetConfigName("app") - viper.SetConfigType("env") - - viper.AutomaticEnv() - - err = viper.ReadInConfig() - if err != nil { - return - } - - err = viper.Unmarshal(&config) - return -}