Crawl dữ liệu tỷ giá hối đoái với AWS Lambda và DynamoDB (Over engineer version)

sam init --name currency-crawler --runtime python3.9

vi ~/.aws/credentials

[vntechies]
aws_access_key_id=******
aws_secret_access_key=******

AWSTemplateFormatVersion: "2010-09-09"
Transform: AWS::Serverless-2016-10-31
Description: >
  currency-crawler

  Sample SAM Template for currency-crawler

# More info about Globals: https://github.com/awslabs/serverless-application-model/blob/master/docs/globals.rst
Globals:
  Function:
    Timeout: 100

Resources:
  CurrencyCrawlFunction:
    Type: AWS::Serverless::Function # More info about Function Resource: https://github.com/awslabs/serverless-application-model/blob/master/versions/2016-10-31.md#awsserverlessfunction
    Properties:
      CodeUri: currency/
      Handler: app.crawl
      Runtime: python3.9
      Architectures:
        - x86_64
      Events:
        InvocationLevel:
          Type: Schedule
          Properties:
            Schedule: cron(0 11 ? * MON-FRI *) # All scheduled events use UTC+0 time zone
            DeadLetterConfig:
              Type: SQS
      Policies:
        DynamoDBWritePolicy:
          TableName: !Ref CurrencyTable

  CurrencyGetFunction:
    Type: AWS::Serverless::Function
    Properties:
      CodeUri: currency/
      Handler: app.get_csv
      Runtime: python3.9
      Architectures:
        - x86_64
      Events:
        CurrencyCrawler:
          Type: HttpApi # More info about API Event Source: https://github.com/awslabs/serverless-application-model/blob/master/versions/2016-10-31.md#api
          Properties:
            Path: /csv
            Method: get
            ApiId: !Ref CurrencyHTTPApi
      Policies:
        DynamoDBReadPolicy:
          TableName: !Ref CurrencyTable

  CurrencyHTTPApi:
    Type: AWS::Serverless::HttpApi
    Properties:
      CorsConfiguration:
        AllowOrigins:
          - "*"
        AllowHeaders:
          - "Content-Transfer-Encoding"
        AllowMethods:
          - GET

  CurrencyTable:
    Type: AWS::Serverless::SimpleTable
    Properties:
      PrimaryKey:
        Name: dt
        Type: String
      #ProvisionedThroughput:
      TableName: "Currency"

Outputs:
  # https://github.com/awslabs/serverless-application-model/blob/master/docs/internals/generated_resources.rst#api
  CurrencyCrawlApi:
    Description: "API Gateway endpoint URL for Prod stage for Hello World function"
    Value: !Sub "https://${CurrencyHTTPApi}.execute-api.${AWS::Region}.amazonaws.com/crawl/"
  CurrencyCrawlFunction:
    Description: "Hello World Lambda Function ARN"
    Value: !GetAtt CurrencyCrawlFunction.Arn
  CurrencyCrawlFunctionIamRole:
    Description: "Implicit IAM Role created for Hello World function"
    Value: !GetAtt CurrencyCrawlFunctionRole.Arn

# test function bằng event mẫu từ EventBridge
sam local invoke "CurrencyCrawlFunction" -e events/event.json
# validate SAM template
sam validate --profile vntechies --region ap-southeast-1
# build package
sam build
# deploy
sam deploy --guided --profile vntechies

bs4
pandas
selenium
webdriver_manager
requests

cd crawl_old_data
pip install -r requirements.txt
vi scb.py

from decimal import Decimal
from time import sleep
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.support.ui import WebDriverWait
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from datetime import date, datetime, timedelta
from bs4 import BeautifulSoup
from selenium.common.exceptions import TimeoutException
import pandas as pd

import pdb
CURRENCIES = ['USD', 'EUR', 'JPY', 'AUD', 'HKD', 'SGD']


def get_record(requested_date_str, driver):
    driver.find_element(By.CSS_SELECTOR, "#dtpNgayMoney > span > i").click()
    title = driver.find_element(
        By.CSS_SELECTOR, "ul > li > div > div > table > thead> tr > th:nth-child(2)")
    previous_button = driver.find_element(
        By.CSS_SELECTOR, "ul > li > div > div > table > thead> tr > th:nth-child(1)")
    next_button = driver.find_element(
        By.CSS_SELECTOR, "ul > li > div > div > table > thead> tr > th:nth-child(3)")
    current_month = int(title.text.split(' ')[1])
    requested_date = datetime.strptime(requested_date_str, '%d/%m/%Y').date()
    while requested_date.month < current_month:
        previous_button.click()
        current_month = int(title.text.split(' ')[1])
    while requested_date.month > current_month:
        next_button.click()
        current_month = int(title.text.split(' ')[1])
    cal_body = driver.find_element(
        By.CSS_SELECTOR, "div.datepicker-days > table > tbody")
    cal_body.find_element(
        By.XPATH, f"//td[text()='{requested_date.day}']").click()
    sleep(5)
    delay = 3  # seconds
    try:
        WebDriverWait(driver, delay).until(
            EC.presence_of_element_located((By.CSS_SELECTOR, '#bdUSDG7 > table')))
        print("Page is ready!")
    except TimeoutException:
        print("Loading took too much time!")
    table = driver.find_element(By.CSS_SELECTOR, "#bdUSDG7 > table")
    content = table.get_attribute('innerHTML')
    soup = BeautifulSoup(content, features="html.parser")
    items = soup.findAll("tr", {"class": "tr-items"})
    record = {
        "bank": "STB",
        "date": requested_date.strftime("%Y-%m-%d")
    }
    for item in items:
        currency = item.find(
            "td", {"class": "td-cell01"}).contents[1].replace(' ', '')
        if currency in CURRENCIES:
            rate = item.find("td", {"class": "td-cell04"}
                             ).contents[0].replace('.', '').replace(',', '.')
            record[currency] = Decimal(rate)
    return record


end_date = date.today()
current_date = end_date + timedelta(days=-150)
delta = timedelta(days=1)

docs = []
driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()))
driver.get("https://www.**.com.vn/company/Pages/ty-gia.aspx")

while current_date <= end_date:
    if current_date.weekday() > 4:
        print("Skip weekends")
        current_date += delta
        continue
    current_date_str = current_date.strftime("%d/%m/%Y")
    print(f"getting data for {current_date_str}")
    docs.append(get_record(current_date_str, driver))
    current_date_str
    current_date += delta
    sleep(2)
df = pd.DataFrame(docs)
df.to_csv('scb_rates.csv', index=False, header=True)

print("Completed!")

python scb.py
python mbb.py
python bidv.py