Code cleaning

This commit is contained in:
Riccardo
2022-07-25 19:05:37 +02:00
parent b7eb930bd8
commit 9f5d892e5b
4 changed files with 36 additions and 19 deletions

View File

@@ -2,3 +2,11 @@
Command to build the executable for manual upload to Lambda: Command to build the executable for manual upload to Lambda:
GOOS=linux CGO_ENABLED=0 go build -o main . GOOS=linux CGO_ENABLED=0 go build -o main .
Required variables:
- FETCH_SIZE: how many news to fetch
- BATCH_SIZE: batch size for DB write
- REGION: AWS region
- SENDER: sender email address
- RECIPIENT: recipient email address

View File

@@ -70,8 +70,10 @@ func CreateTable(basics Table) (*types.TableDescription, error) {
if err != nil { if err != nil {
log.Printf("Wait for table exists failed. Here's why: %v\n", err) log.Printf("Wait for table exists failed. Here's why: %v\n", err)
} }
tableDesc = table.TableDescription tableDesc = table.TableDescription
} }
return tableDesc, err return tableDesc, err
} }
@@ -87,19 +89,24 @@ func AddNewsBatch(basics Table, news []News, batchSize int) (int, error) {
written := 0 written := 0
start := 0 start := 0
end := start + batchSize end := start + batchSize
for start < len(news) { for start < len(news) {
var writeReqs []types.WriteRequest var writeReqs []types.WriteRequest
if end > len(news) { if end > len(news) {
end = len(news) end = len(news)
} }
for _, entry := range news[start:end] { for _, entry := range news[start:end] {
item, err = attributevalue.MarshalMap(entry) item, err = attributevalue.MarshalMap(entry)
if err != nil { if err != nil {
log.Printf("Couldn't marshal news %v for batch writing: %v\n", entry.Title, err) log.Printf("Couldn't marshal news %v for batch writing: %v\n", entry.Title, err)
} else { } else {
writeReqs = append(writeReqs, types.WriteRequest{PutRequest: &types.PutRequest{Item: item}}) writeReqs = append(writeReqs, types.WriteRequest{PutRequest: &types.PutRequest{Item: item}})
} }
} }
_, err = basics.DynamoDbClient.BatchWriteItem(context.TODO(), &dynamodb.BatchWriteItemInput{ _, err = basics.DynamoDbClient.BatchWriteItem(context.TODO(), &dynamodb.BatchWriteItemInput{
RequestItems: map[string][]types.WriteRequest{basics.TableName: writeReqs}}) RequestItems: map[string][]types.WriteRequest{basics.TableName: writeReqs}})
if err != nil { if err != nil {
@@ -107,6 +114,7 @@ func AddNewsBatch(basics Table, news []News, batchSize int) (int, error) {
} else { } else {
written += len(writeReqs) written += len(writeReqs)
} }
start = end start = end
end += batchSize end += batchSize
} }
@@ -120,7 +128,7 @@ func ReadTodayNews(basics Table, timeStart int, timeEnd int) ([]News, error) {
var news []News var news []News
params, err := attributevalue.MarshalList([]interface{}{timeStart, timeEnd}) params, err := attributevalue.MarshalList([]interface{}{timeStart, timeEnd})
if err != nil { if err != nil {
panic(err) log.Fatalln(err)
} }
response, err := basics.DynamoDbClient.ExecuteStatement(context.TODO(), &dynamodb.ExecuteStatementInput{ response, err := basics.DynamoDbClient.ExecuteStatement(context.TODO(), &dynamodb.ExecuteStatementInput{

View File

@@ -23,8 +23,8 @@ func GetTopNewsIds(url string) (response []string) {
log.Fatalln(err) log.Fatalln(err)
} }
body_string := string(body) bodyString := string(body)
response = strings.Split(body_string[1:len(body_string)-1], ",") response = strings.Split(bodyString[1:len(bodyString)-1], ",")
fmt.Println("Done.") fmt.Println("Done.")

View File

@@ -8,6 +8,9 @@ import (
"hackernewsletter/hackernews" "hackernewsletter/hackernews"
"hackernewsletter/mail" "hackernewsletter/mail"
"html/template" "html/template"
"log"
"os"
"strconv"
"time" "time"
"github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/config"
@@ -15,11 +18,15 @@ import (
) )
func Handler(ctx context.Context) (string, error) { func Handler(ctx context.Context) (string, error) {
batchSize, _ := strconv.Atoi(os.Getenv("BATCH_SIZE"))
fetchSize, _ := strconv.Atoi(os.Getenv("FETCH_SIZE"))
var newsBatch []db.News
table := "news_table" table := "news_table"
cfg, err := config.LoadDefaultConfig(context.TODO()) cfg, err := config.LoadDefaultConfig(context.TODO())
if err != nil { if err != nil {
panic("Unable to load SDK config, " + err.Error()) log.Fatalln("Unable to load SDK config, " + err.Error())
} }
client := dynamodb.NewFromConfig(cfg) client := dynamodb.NewFromConfig(cfg)
@@ -35,7 +42,7 @@ func Handler(ctx context.Context) (string, error) {
println(("Table not found. Creating it...")) println(("Table not found. Creating it..."))
_, new_err := db.CreateTable(newsTable) _, new_err := db.CreateTable(newsTable)
if new_err != nil { if new_err != nil {
panic("Failed creating table " + table) log.Fatalln("Failed creating table.")
} }
resp, err = db.GetTableInfo(context.TODO(), client, input) resp, err = db.GetTableInfo(context.TODO(), client, input)
@@ -45,14 +52,12 @@ func Handler(ctx context.Context) (string, error) {
newsIds := hackernews.GetTopNewsIds("https://hacker-news.firebaseio.com/v0/topstories.json") newsIds := hackernews.GetTopNewsIds("https://hacker-news.firebaseio.com/v0/topstories.json")
var newsBatch []db.News for i := 0; i < fetchSize; i++ {
for i := 0; i < 10; i++ {
myNews := hackernews.GetNewsById(newsIds[i], "https://hacker-news.firebaseio.com/v0/item/{ID}.json") myNews := hackernews.GetNewsById(newsIds[i], "https://hacker-news.firebaseio.com/v0/item/{ID}.json")
newsBatch = append(newsBatch, myNews) newsBatch = append(newsBatch, myNews)
} }
db.AddNewsBatch(newsTable, newsBatch, 25) db.AddNewsBatch(newsTable, newsBatch, batchSize)
timeEnd := time.Date(time.Now().Year(), time.Now().Month(), time.Now().Day(), 0, 0, 0, 0, time.UTC) timeEnd := time.Date(time.Now().Year(), time.Now().Month(), time.Now().Day(), 0, 0, 0, 0, time.UTC)
timeStart := timeEnd.Add(-time.Hour * 24) timeStart := timeEnd.Add(-time.Hour * 24)
@@ -60,28 +65,24 @@ func Handler(ctx context.Context) (string, error) {
todayNews, _ := db.ReadTodayNews(newsTable, int(timeStart.Unix()), int(timeEnd.Unix())) todayNews, _ := db.ReadTodayNews(newsTable, int(timeStart.Unix()), int(timeEnd.Unix()))
t, err := template.ParseFiles("mail/index.gohtml") t, err := template.ParseFiles("mail/index.gohtml")
if err != nil { if err != nil {
panic(err) log.Fatalln(err)
} }
data := struct { emailNews := struct {
News []db.News News []db.News
}{ }{
News: todayNews, News: todayNews,
} }
var doc bytes.Buffer var emailNewsBuffer bytes.Buffer
err = t.Execute(&doc, data)
err = t.Execute(&emailNewsBuffer, emailNews)
if err != nil { if err != nil {
panic(err) log.Fatalln(err)
} }
body := doc.String() mail.SendNewsletter(emailNewsBuffer.String())
mail.SendNewsletter(body)
return "Completed.", err return "Completed.", err
} }