riccardo/hackernews-lambda-on-aws
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added code for AWS Lambda

Browse Source
This commit is contained in:
Riccardo
2022-07-16 16:37:07 +02:00
parent a922a905c0
commit 98273e164d
6 changed files with 74 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
db/db.go
View File

@@ -15,7 +15,6 @@ import (
type Table struct {
DynamoDbClient *dynamodb.Client
TableName string
IndexName string
}
type News struct {
@@ -139,7 +138,9 @@ func ReadTodayNews(basics Table, timeStart int, timeEnd int) ([]News, error) {
}
}
fmt.Println("MARSH", len(response.Items), news[0])
if len(response.Items) > 0 {
fmt.Println("MARSH", len(response.Items), news[0])
}
return news, err
}

4
go.mod
View File

@@ -3,10 +3,10 @@ module hackernewsletter
go 1.18
require (
github.com/aws/aws-lambda-go v1.32.1
github.com/aws/aws-sdk-go-v2 v1.16.7
github.com/aws/aws-sdk-go-v2/config v1.15.14
github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.9.6
github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression v1.4.12
github.com/aws/aws-sdk-go-v2/service/dynamodb v1.15.9
github.com/spf13/viper v1.12.0
)
@@ -40,5 +40,5 @@ require (
golang.org/x/text v0.3.7 // indirect
gopkg.in/ini.v1 v1.66.4 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

10
go.sum
View File

@@ -38,6 +38,8 @@ cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3f
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/aws/aws-lambda-go v1.32.1 h1:ls0FU8Mt7ayJszb945zFkUfzxhkQTli8mpJstVcDtCY=
github.com/aws/aws-lambda-go v1.32.1/go.mod h1:jwFe2KmMsHmffA1X2R09hH6lFzJQxzI8qK17ewzbQMM=
github.com/aws/aws-sdk-go-v2 v1.16.7 h1:zfBwXus3u14OszRxGcqCDS4MfMCv10e8SMJ2r8Xm0Ns=
github.com/aws/aws-sdk-go-v2 v1.16.7/go.mod h1:6CpKuLXg2w7If3ABZCl/qZ6rEgwtjZTn4eAf4RcEyuw=
github.com/aws/aws-sdk-go-v2/config v1.15.14 h1:+BqpqlydTq4c2et9Daury7gE+o67P4lbk7eybiCBNc4=
@@ -46,8 +48,6 @@ github.com/aws/aws-sdk-go-v2/credentials v1.12.9 h1:DloAJr0/jbvm0iVRFDFh8GlWxrOd
github.com/aws/aws-sdk-go-v2/credentials v1.12.9/go.mod h1:2Vavxl1qqQXJ8MUcQZTsIEW8cwenFCWYXtLRPba3L/o=
github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.9.6 h1:vlEfSyZ2pZjOZe7zsPIAFem17w2HeeFULk7TPVWoDR4=
github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.9.6/go.mod h1:+/KXTIzLmrjdlQVgiE14/jhy9GyDZnmMGQoykod99Lw=
github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression v1.4.12 h1:sH0SffGPiNpvYCCfEF0dN0K9OC72KXBjW4HmiFvMVf0=
github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression v1.4.12/go.mod h1:0vvQ0FQRjyNB8EIkRdwT9tduJbkUdh00SnmuKnZRYLA=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.8 h1:VfBdn2AxwMbFyJN/lF/xuT3SakomJ86PZu3rCxb5K0s=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.8/go.mod h1:oL1Q3KuCq1D4NykQnIvtRiBGLUXhcpY5pl6QZB2XEPU=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.14 h1:2C0pYHcUBmdzPj+EKNC4qj97oK6yjrUhc1KoSodglvk=
@@ -200,8 +200,8 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.2 h1:4jaiDzPyXQvSd7D0EjG45355tLlV3VOECpq10pLC+8s=
github.com/subosito/gotenv v1.3.0 h1:mjC+YW8QpAdXibNi+vNWgzmgBH4+5l5dCXv8cNysBLI=
github.com/subosito/gotenv v1.3.0/go.mod h1:YzJjq/33h7nrwdY+iHMhEOEEbW0ovIz0tB6t6PwAXzs=
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -503,8 +503,8 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0 h1:hjy8E9ON/egN1tAYqKb61G10WtihqetD4sz2H+8nIeA=
gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

61
handler/handler.go Normal file
View File

@@ -0,0 +1,61 @@
package handler
import (
"context"
"fmt"
"hackernewsletter/db"
"hackernewsletter/hackernews"
"time"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/service/dynamodb"
)
func Handler(ctx context.Context) (string, error) {
table := "news_table"
cfg, err := config.LoadDefaultConfig(context.TODO())
if err != nil {
panic("Unable to load SDK config, " + err.Error())
}
client := dynamodb.NewFromConfig(cfg)
input := &dynamodb.DescribeTableInput{
TableName: &table,
}
newsTable := db.Table{client, table}
resp, err := db.GetTableInfo(context.TODO(), client, input)
if err != nil {
println(("Table not found. Creating it..."))
_, new_err := db.CreateTable(newsTable)
if new_err != nil {
panic("Failed creating table " + table)
}
resp, err = db.GetTableInfo(context.TODO(), client, input)
}
fmt.Printf("Table %v has %v elements\n", table, resp.Table.ItemCount)
newsIds := hackernews.GetTopNewsIds("https://hacker-news.firebaseio.com/v0/topstories.json")
var newsBatch []db.News
for i := 0; i < 10; i++ {
myNews := hackernews.GetNewsById(newsIds[i], "https://hacker-news.firebaseio.com/v0/item/{ID}.json")
newsBatch = append(newsBatch, myNews)
}
db.AddNewsBatch(newsTable, newsBatch, 25)
timeEnd := time.Date(time.Now().Year(), time.Now().Month(), time.Now().Day(), 0, 0, 0, 0, time.UTC)
timeStart := timeEnd.Add(-time.Hour * 24)
todayNews, _ := db.ReadTodayNews(newsTable, int(timeStart.Unix()), int(timeEnd.Unix()))
fmt.Print("NEWS", todayNews)
return "Completed.", err
}

68
main.go
View File

@@ -1,73 +1,11 @@
package main
import (
"context"
"flag"
"fmt"
"hackernewsletter/db"
"hackernewsletter/hackernews"
"hackernewsletter/params"
"log"
"time"
"hackernewsletter/handler"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/service/dynamodb"
"github.com/aws/aws-lambda-go/lambda"
)
func main() {
conf, err := params.LoadConfig(".")
if err != nil {
log.Fatal("Cannot load config:", err)
}
table := flag.String("t", "", "The name of the table")
flag.Parse()
if *table == "" {
fmt.Println("Table name not specified. Using default name.")
table = &conf.TableName
}
cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithRegion(conf.AwsRegion))
if err != nil {
panic("Unable to load SDK config, " + err.Error())
}
client := dynamodb.NewFromConfig(cfg)
input := &dynamodb.DescribeTableInput{
TableName: table,
}
newsTable := db.Table{client, *table, conf.TableIndex}
resp, err := db.GetTableInfo(context.TODO(), client, input)
if err != nil {
println(("Table not found. Creating it..."))
_, new_err := db.CreateTable(newsTable)
if new_err != nil {
panic("Failed creating table " + *table)
}
resp, err = db.GetTableInfo(context.TODO(), client, input)
}
fmt.Printf("Table %v has %v elements\n", *table, resp.Table.ItemCount)
newsIds := hackernews.GetTopNewsIds(conf.TopNews)
var newsBatch []db.News
for i := 0; i < 10; i++ {
myNews := hackernews.GetNewsById(newsIds[i], conf.SingleNews)
newsBatch = append(newsBatch, myNews)
}
db.AddNewsBatch(newsTable, newsBatch, conf.BatchSize)
timeEnd := time.Date(time.Now().Year(), time.Now().Month(), time.Now().Day(), 0, 0, 0, 0, time.UTC)
timeStart := timeEnd.Add(-time.Hour * 24)
todayNews, _ := db.ReadTodayNews(newsTable, int(timeStart.Unix()), int(timeEnd.Unix()))
println("NEWS", todayNews)
lambda.Start(handler.Handler)
}

28
params/params.go
View File

@@ -1,28 +0,0 @@
package params
import "github.com/spf13/viper"
type Config struct {
TableName string `mapstructure:"TABLE_NAME"`
TableIndex string `mapstructure:"GLOBAL_SECONDARY_INDEX"`
AwsRegion string `mapstructure:"AWS_REGION"`
TopNews string `mapstructure:"TOP_NEWS"`
SingleNews string `mapstructure:"SINGLE_NEWS"`
BatchSize int `mapstructure:"SERVER_ADDRESS"`
}
func LoadConfig(path string) (config Config, err error) {
viper.AddConfigPath(path)
viper.SetConfigName("app")
viper.SetConfigType("env")
viper.AutomaticEnv()
err = viper.ReadInConfig()
if err != nil {
return
}
err = viper.Unmarshal(&config)
return
}