diff --git a/README.md b/README.md index 1944062..59069af 100644 --- a/README.md +++ b/README.md @@ -2,3 +2,11 @@ Command to build the executable for manual upload to Lambda: GOOS=linux CGO_ENABLED=0 go build -o main . + +Required variables: + +- FETCH_SIZE: how many news to fetch +- BATCH_SIZE: batch size for DB write +- REGION: AWS region +- SENDER: sender email address +- RECIPIENT: recipient email address diff --git a/db/db.go b/db/db.go index 205032a..268a113 100644 --- a/db/db.go +++ b/db/db.go @@ -70,8 +70,10 @@ func CreateTable(basics Table) (*types.TableDescription, error) { if err != nil { log.Printf("Wait for table exists failed. Here's why: %v\n", err) } + tableDesc = table.TableDescription } + return tableDesc, err } @@ -87,19 +89,24 @@ func AddNewsBatch(basics Table, news []News, batchSize int) (int, error) { written := 0 start := 0 end := start + batchSize + for start < len(news) { var writeReqs []types.WriteRequest + if end > len(news) { end = len(news) } + for _, entry := range news[start:end] { item, err = attributevalue.MarshalMap(entry) + if err != nil { log.Printf("Couldn't marshal news %v for batch writing: %v\n", entry.Title, err) } else { writeReqs = append(writeReqs, types.WriteRequest{PutRequest: &types.PutRequest{Item: item}}) } } + _, err = basics.DynamoDbClient.BatchWriteItem(context.TODO(), &dynamodb.BatchWriteItemInput{ RequestItems: map[string][]types.WriteRequest{basics.TableName: writeReqs}}) if err != nil { @@ -107,6 +114,7 @@ func AddNewsBatch(basics Table, news []News, batchSize int) (int, error) { } else { written += len(writeReqs) } + start = end end += batchSize } @@ -120,7 +128,7 @@ func ReadTodayNews(basics Table, timeStart int, timeEnd int) ([]News, error) { var news []News params, err := attributevalue.MarshalList([]interface{}{timeStart, timeEnd}) if err != nil { - panic(err) + log.Fatalln(err) } response, err := basics.DynamoDbClient.ExecuteStatement(context.TODO(), &dynamodb.ExecuteStatementInput{ diff --git a/hackernews/hackernews.go b/hackernews/hackernews.go index ba990c1..cf30497 100644 --- a/hackernews/hackernews.go +++ b/hackernews/hackernews.go @@ -23,8 +23,8 @@ func GetTopNewsIds(url string) (response []string) { log.Fatalln(err) } - body_string := string(body) - response = strings.Split(body_string[1:len(body_string)-1], ",") + bodyString := string(body) + response = strings.Split(bodyString[1:len(bodyString)-1], ",") fmt.Println("Done.") diff --git a/handler/handler.go b/handler/handler.go index 096cc34..d9f7886 100644 --- a/handler/handler.go +++ b/handler/handler.go @@ -8,6 +8,9 @@ import ( "hackernewsletter/hackernews" "hackernewsletter/mail" "html/template" + "log" + "os" + "strconv" "time" "github.com/aws/aws-sdk-go-v2/config" @@ -15,11 +18,15 @@ import ( ) func Handler(ctx context.Context) (string, error) { + batchSize, _ := strconv.Atoi(os.Getenv("BATCH_SIZE")) + fetchSize, _ := strconv.Atoi(os.Getenv("FETCH_SIZE")) + + var newsBatch []db.News table := "news_table" cfg, err := config.LoadDefaultConfig(context.TODO()) if err != nil { - panic("Unable to load SDK config, " + err.Error()) + log.Fatalln("Unable to load SDK config, " + err.Error()) } client := dynamodb.NewFromConfig(cfg) @@ -35,7 +42,7 @@ func Handler(ctx context.Context) (string, error) { println(("Table not found. Creating it...")) _, new_err := db.CreateTable(newsTable) if new_err != nil { - panic("Failed creating table " + table) + log.Fatalln("Failed creating table.") } resp, err = db.GetTableInfo(context.TODO(), client, input) @@ -45,14 +52,12 @@ func Handler(ctx context.Context) (string, error) { newsIds := hackernews.GetTopNewsIds("https://hacker-news.firebaseio.com/v0/topstories.json") - var newsBatch []db.News - - for i := 0; i < 10; i++ { + for i := 0; i < fetchSize; i++ { myNews := hackernews.GetNewsById(newsIds[i], "https://hacker-news.firebaseio.com/v0/item/{ID}.json") newsBatch = append(newsBatch, myNews) } - db.AddNewsBatch(newsTable, newsBatch, 25) + db.AddNewsBatch(newsTable, newsBatch, batchSize) timeEnd := time.Date(time.Now().Year(), time.Now().Month(), time.Now().Day(), 0, 0, 0, 0, time.UTC) timeStart := timeEnd.Add(-time.Hour * 24) @@ -60,28 +65,24 @@ func Handler(ctx context.Context) (string, error) { todayNews, _ := db.ReadTodayNews(newsTable, int(timeStart.Unix()), int(timeEnd.Unix())) t, err := template.ParseFiles("mail/index.gohtml") - if err != nil { - panic(err) + log.Fatalln(err) } - data := struct { + emailNews := struct { News []db.News }{ News: todayNews, } - var doc bytes.Buffer - - err = t.Execute(&doc, data) + var emailNewsBuffer bytes.Buffer + err = t.Execute(&emailNewsBuffer, emailNews) if err != nil { - panic(err) + log.Fatalln(err) } - body := doc.String() - - mail.SendNewsletter(body) + mail.SendNewsletter(emailNewsBuffer.String()) return "Completed.", err }