136 lines
5.7 KiB
Python
136 lines
5.7 KiB
Python
|
import sys
|
||
|
|
import requests
|
||
|
|
from dbfread import DBF
|
||
|
|
from datetime import date, datetime
|
||
|
|
import csv
|
||
|
|
|
||
|
|
import os
|
||
|
|
from dotenv import load_dotenv
|
||
|
|
load_dotenv(override=True)
|
||
|
|
host=os.getenv("HOST")
|
||
|
|
login_user = os.getenv("USER")
|
||
|
|
login_pass = os.getenv("PASS")
|
||
|
|
src_file_subs = os.getenv("FILE_ROOT")+os.getenv("SUBSCRIPTION_FILE")
|
||
|
|
print("Reading files: ", src_file_subs)
|
||
|
|
|
||
|
|
def read_dbf_file(file_path, columns):
|
||
|
|
data = []
|
||
|
|
try:
|
||
|
|
# Open the .dbf file
|
||
|
|
with DBF(file_path) as dbf:
|
||
|
|
# Iterate over each record in the .dbf file
|
||
|
|
for record in dbf:
|
||
|
|
# Filter the record to include only the specified columns
|
||
|
|
filtered_record = {column: record.get(column) for column in columns}
|
||
|
|
|
||
|
|
# Clean up the ACNO field by removing spaces and keeping only numeric characters
|
||
|
|
if filtered_record.get('ACNO'):
|
||
|
|
filtered_record['ACNO'] = ''.join(filter(str.isdigit, filtered_record['ACNO']))
|
||
|
|
|
||
|
|
# Clean up the PHONE field by keeping only numeric characters
|
||
|
|
if filtered_record.get('PHONE'):
|
||
|
|
filtered_record['PHONE'] = ''.join(filter(str.isdigit, filtered_record['PHONE']))
|
||
|
|
|
||
|
|
# Check if NAME2 is empty
|
||
|
|
if not filtered_record.get('NAME2'):
|
||
|
|
# If NAME2 is empty, check if PHONE has a value
|
||
|
|
if filtered_record.get('PHONE'):
|
||
|
|
filtered_record['NAME2'] = filtered_record['PHONE']
|
||
|
|
else:
|
||
|
|
# If both NAME2 and PHONE are empty, skip this record
|
||
|
|
continue
|
||
|
|
|
||
|
|
# Check if PHONE length is not equal to 10
|
||
|
|
if len(filtered_record.get('PHONE', '')) != 10:
|
||
|
|
continue
|
||
|
|
|
||
|
|
# Merge [ADD1, ADD2, ADD3, PLACE, PIN] into one column "Address"
|
||
|
|
address_components = [record.get(col, '') for col in ['ADD1', 'ADD2', 'ADD3', 'PLACE', 'PIN']]
|
||
|
|
filtered_record['Address'] = '-'.join(filter(None, address_components))
|
||
|
|
|
||
|
|
# Remove [ADD1, ADD2, ADD3, PLACE, PIN]
|
||
|
|
for col in ['ADD1', 'ADD2', 'ADD3', 'PLACE', 'PIN']:
|
||
|
|
del filtered_record[col]
|
||
|
|
|
||
|
|
data.append(filtered_record)
|
||
|
|
except Exception as e:
|
||
|
|
print("Error:", e)
|
||
|
|
return data
|
||
|
|
|
||
|
|
def save_data_to_csv(data, file_path, columns):
|
||
|
|
try:
|
||
|
|
# Remove unnecessary columns from the fieldnames
|
||
|
|
columns = [col for col in columns if col not in ["ADD1", "ADD2", "ADD3", "PLACE", "PIN", 'Address', 'NAME2']]
|
||
|
|
|
||
|
|
# Remove unnecessary columns from each record in the data
|
||
|
|
for record in data:
|
||
|
|
for field in ["ADD1", "ADD2", "ADD3", "PLACE", "PIN", 'Address', 'NAME2']:
|
||
|
|
if field in record:
|
||
|
|
del record[field]
|
||
|
|
|
||
|
|
with open(file_path, 'w', newline='') as csv_file:
|
||
|
|
writer = csv.DictWriter(csv_file, fieldnames=columns)
|
||
|
|
writer.writeheader()
|
||
|
|
writer.writerows(data)
|
||
|
|
print(f"CSV saved to {file_path}")
|
||
|
|
except Exception as e:
|
||
|
|
print("Error saving CSV to file:", e)
|
||
|
|
|
||
|
|
def get_auth_token():
|
||
|
|
url = host+"/auth/token/"
|
||
|
|
print("Using URL: ", url)
|
||
|
|
payload = {"phone_no": login_user, "password": login_pass}
|
||
|
|
try:
|
||
|
|
response = requests.post(url, data=payload)
|
||
|
|
response.raise_for_status()
|
||
|
|
return response.json().get('access')
|
||
|
|
except requests.exceptions.RequestException as e:
|
||
|
|
print("Error obtaining auth token:", e)
|
||
|
|
return None
|
||
|
|
|
||
|
|
def send_data_to_api(data, token, batch_size=1000):
|
||
|
|
url = host+"/api/v1/migrateSubscriptions"
|
||
|
|
print("Using URL: ", url)
|
||
|
|
csv_file_path = 'batch.csv'
|
||
|
|
headers = {'Authorization': f'Bearer {token}'}
|
||
|
|
for i in range(0, len(data), batch_size):
|
||
|
|
batch = data[i:i+batch_size]
|
||
|
|
save_data_to_csv(batch, csv_file_path, batch[0].keys())
|
||
|
|
try:
|
||
|
|
# Send CSV file with data as multipart form-data
|
||
|
|
files = {'file': open(csv_file_path, 'rb')}
|
||
|
|
response = requests.post(url, files=files, headers=headers)
|
||
|
|
|
||
|
|
# Check response status
|
||
|
|
if response.status_code == 201:
|
||
|
|
response_data = response.json()
|
||
|
|
# Print the message
|
||
|
|
print(f"Message from server: {response_data.get('message')}")
|
||
|
|
print(f"Batch {i//batch_size + 1} sent successfully to the API")
|
||
|
|
else:
|
||
|
|
# Print the error message and response content
|
||
|
|
print(f"Failed to send batch {i//batch_size + 1} to the API. Status code:", response.status_code)
|
||
|
|
print("Response content:", response.content.decode('utf-8'))
|
||
|
|
except Exception as e:
|
||
|
|
# Print the general error message
|
||
|
|
print("Error:", e)
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
# Check if the file path is provided as a command-line argument
|
||
|
|
# if len(sys.argv) < 2:
|
||
|
|
# print("Usage: python script.py DE2.DBF")
|
||
|
|
# else:
|
||
|
|
# Get the file path from the command-line arguments
|
||
|
|
dbf_file_path = src_file_subs
|
||
|
|
# Specify the columns to be extracted
|
||
|
|
columns_to_extract = ["ACTP", "ACNO", "PHONE", "ADD1", "ADD2", "ADD3", "PLACE", "PIN"]
|
||
|
|
# Call the function to read the .dbf file with specific columns
|
||
|
|
dbf_data = read_dbf_file(dbf_file_path, columns_to_extract)
|
||
|
|
# Obtain the authentication token
|
||
|
|
token = get_auth_token()
|
||
|
|
if token:
|
||
|
|
# Send the data to the API in batches with authentication
|
||
|
|
send_data_to_api(dbf_data, token)
|
||
|
|
else:
|
||
|
|
print("Failed to obtain auth token, cannot send data to API.")
|