Code cleaning
This commit is contained in:
@@ -2,3 +2,11 @@
|
|||||||
|
|
||||||
Command to build the executable for manual upload to Lambda:
|
Command to build the executable for manual upload to Lambda:
|
||||||
GOOS=linux CGO_ENABLED=0 go build -o main .
|
GOOS=linux CGO_ENABLED=0 go build -o main .
|
||||||
|
|
||||||
|
Required variables:
|
||||||
|
|
||||||
|
- FETCH_SIZE: how many news to fetch
|
||||||
|
- BATCH_SIZE: batch size for DB write
|
||||||
|
- REGION: AWS region
|
||||||
|
- SENDER: sender email address
|
||||||
|
- RECIPIENT: recipient email address
|
||||||
|
|||||||
10
db/db.go
10
db/db.go
@@ -70,8 +70,10 @@ func CreateTable(basics Table) (*types.TableDescription, error) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
log.Printf("Wait for table exists failed. Here's why: %v\n", err)
|
log.Printf("Wait for table exists failed. Here's why: %v\n", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
tableDesc = table.TableDescription
|
tableDesc = table.TableDescription
|
||||||
}
|
}
|
||||||
|
|
||||||
return tableDesc, err
|
return tableDesc, err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -87,19 +89,24 @@ func AddNewsBatch(basics Table, news []News, batchSize int) (int, error) {
|
|||||||
written := 0
|
written := 0
|
||||||
start := 0
|
start := 0
|
||||||
end := start + batchSize
|
end := start + batchSize
|
||||||
|
|
||||||
for start < len(news) {
|
for start < len(news) {
|
||||||
var writeReqs []types.WriteRequest
|
var writeReqs []types.WriteRequest
|
||||||
|
|
||||||
if end > len(news) {
|
if end > len(news) {
|
||||||
end = len(news)
|
end = len(news)
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, entry := range news[start:end] {
|
for _, entry := range news[start:end] {
|
||||||
item, err = attributevalue.MarshalMap(entry)
|
item, err = attributevalue.MarshalMap(entry)
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Printf("Couldn't marshal news %v for batch writing: %v\n", entry.Title, err)
|
log.Printf("Couldn't marshal news %v for batch writing: %v\n", entry.Title, err)
|
||||||
} else {
|
} else {
|
||||||
writeReqs = append(writeReqs, types.WriteRequest{PutRequest: &types.PutRequest{Item: item}})
|
writeReqs = append(writeReqs, types.WriteRequest{PutRequest: &types.PutRequest{Item: item}})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err = basics.DynamoDbClient.BatchWriteItem(context.TODO(), &dynamodb.BatchWriteItemInput{
|
_, err = basics.DynamoDbClient.BatchWriteItem(context.TODO(), &dynamodb.BatchWriteItemInput{
|
||||||
RequestItems: map[string][]types.WriteRequest{basics.TableName: writeReqs}})
|
RequestItems: map[string][]types.WriteRequest{basics.TableName: writeReqs}})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -107,6 +114,7 @@ func AddNewsBatch(basics Table, news []News, batchSize int) (int, error) {
|
|||||||
} else {
|
} else {
|
||||||
written += len(writeReqs)
|
written += len(writeReqs)
|
||||||
}
|
}
|
||||||
|
|
||||||
start = end
|
start = end
|
||||||
end += batchSize
|
end += batchSize
|
||||||
}
|
}
|
||||||
@@ -120,7 +128,7 @@ func ReadTodayNews(basics Table, timeStart int, timeEnd int) ([]News, error) {
|
|||||||
var news []News
|
var news []News
|
||||||
params, err := attributevalue.MarshalList([]interface{}{timeStart, timeEnd})
|
params, err := attributevalue.MarshalList([]interface{}{timeStart, timeEnd})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
panic(err)
|
log.Fatalln(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
response, err := basics.DynamoDbClient.ExecuteStatement(context.TODO(), &dynamodb.ExecuteStatementInput{
|
response, err := basics.DynamoDbClient.ExecuteStatement(context.TODO(), &dynamodb.ExecuteStatementInput{
|
||||||
|
|||||||
@@ -23,8 +23,8 @@ func GetTopNewsIds(url string) (response []string) {
|
|||||||
log.Fatalln(err)
|
log.Fatalln(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
body_string := string(body)
|
bodyString := string(body)
|
||||||
response = strings.Split(body_string[1:len(body_string)-1], ",")
|
response = strings.Split(bodyString[1:len(bodyString)-1], ",")
|
||||||
|
|
||||||
fmt.Println("Done.")
|
fmt.Println("Done.")
|
||||||
|
|
||||||
|
|||||||
@@ -8,6 +8,9 @@ import (
|
|||||||
"hackernewsletter/hackernews"
|
"hackernewsletter/hackernews"
|
||||||
"hackernewsletter/mail"
|
"hackernewsletter/mail"
|
||||||
"html/template"
|
"html/template"
|
||||||
|
"log"
|
||||||
|
"os"
|
||||||
|
"strconv"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/aws/aws-sdk-go-v2/config"
|
"github.com/aws/aws-sdk-go-v2/config"
|
||||||
@@ -15,11 +18,15 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
func Handler(ctx context.Context) (string, error) {
|
func Handler(ctx context.Context) (string, error) {
|
||||||
|
batchSize, _ := strconv.Atoi(os.Getenv("BATCH_SIZE"))
|
||||||
|
fetchSize, _ := strconv.Atoi(os.Getenv("FETCH_SIZE"))
|
||||||
|
|
||||||
|
var newsBatch []db.News
|
||||||
table := "news_table"
|
table := "news_table"
|
||||||
|
|
||||||
cfg, err := config.LoadDefaultConfig(context.TODO())
|
cfg, err := config.LoadDefaultConfig(context.TODO())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
panic("Unable to load SDK config, " + err.Error())
|
log.Fatalln("Unable to load SDK config, " + err.Error())
|
||||||
}
|
}
|
||||||
|
|
||||||
client := dynamodb.NewFromConfig(cfg)
|
client := dynamodb.NewFromConfig(cfg)
|
||||||
@@ -35,7 +42,7 @@ func Handler(ctx context.Context) (string, error) {
|
|||||||
println(("Table not found. Creating it..."))
|
println(("Table not found. Creating it..."))
|
||||||
_, new_err := db.CreateTable(newsTable)
|
_, new_err := db.CreateTable(newsTable)
|
||||||
if new_err != nil {
|
if new_err != nil {
|
||||||
panic("Failed creating table " + table)
|
log.Fatalln("Failed creating table.")
|
||||||
}
|
}
|
||||||
|
|
||||||
resp, err = db.GetTableInfo(context.TODO(), client, input)
|
resp, err = db.GetTableInfo(context.TODO(), client, input)
|
||||||
@@ -45,14 +52,12 @@ func Handler(ctx context.Context) (string, error) {
|
|||||||
|
|
||||||
newsIds := hackernews.GetTopNewsIds("https://hacker-news.firebaseio.com/v0/topstories.json")
|
newsIds := hackernews.GetTopNewsIds("https://hacker-news.firebaseio.com/v0/topstories.json")
|
||||||
|
|
||||||
var newsBatch []db.News
|
for i := 0; i < fetchSize; i++ {
|
||||||
|
|
||||||
for i := 0; i < 10; i++ {
|
|
||||||
myNews := hackernews.GetNewsById(newsIds[i], "https://hacker-news.firebaseio.com/v0/item/{ID}.json")
|
myNews := hackernews.GetNewsById(newsIds[i], "https://hacker-news.firebaseio.com/v0/item/{ID}.json")
|
||||||
newsBatch = append(newsBatch, myNews)
|
newsBatch = append(newsBatch, myNews)
|
||||||
}
|
}
|
||||||
|
|
||||||
db.AddNewsBatch(newsTable, newsBatch, 25)
|
db.AddNewsBatch(newsTable, newsBatch, batchSize)
|
||||||
|
|
||||||
timeEnd := time.Date(time.Now().Year(), time.Now().Month(), time.Now().Day(), 0, 0, 0, 0, time.UTC)
|
timeEnd := time.Date(time.Now().Year(), time.Now().Month(), time.Now().Day(), 0, 0, 0, 0, time.UTC)
|
||||||
timeStart := timeEnd.Add(-time.Hour * 24)
|
timeStart := timeEnd.Add(-time.Hour * 24)
|
||||||
@@ -60,28 +65,24 @@ func Handler(ctx context.Context) (string, error) {
|
|||||||
todayNews, _ := db.ReadTodayNews(newsTable, int(timeStart.Unix()), int(timeEnd.Unix()))
|
todayNews, _ := db.ReadTodayNews(newsTable, int(timeStart.Unix()), int(timeEnd.Unix()))
|
||||||
|
|
||||||
t, err := template.ParseFiles("mail/index.gohtml")
|
t, err := template.ParseFiles("mail/index.gohtml")
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
panic(err)
|
log.Fatalln(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
data := struct {
|
emailNews := struct {
|
||||||
News []db.News
|
News []db.News
|
||||||
}{
|
}{
|
||||||
News: todayNews,
|
News: todayNews,
|
||||||
}
|
}
|
||||||
|
|
||||||
var doc bytes.Buffer
|
var emailNewsBuffer bytes.Buffer
|
||||||
|
|
||||||
err = t.Execute(&doc, data)
|
|
||||||
|
|
||||||
|
err = t.Execute(&emailNewsBuffer, emailNews)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
panic(err)
|
log.Fatalln(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
body := doc.String()
|
mail.SendNewsletter(emailNewsBuffer.String())
|
||||||
|
|
||||||
mail.SendNewsletter(body)
|
|
||||||
|
|
||||||
return "Completed.", err
|
return "Completed.", err
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user