import axios from "axios";
import csv from "csv-parser";
import fs from "fs-extra";
import { createReadStream } from "fs";
interface Client {
idNumber: string;
nameEn: string;
nameAr: string;
nationalityCode?: string;
dobGregorian?: string;
// ... other optional fields
}
const config = {
apiEndpoint: "https://<environment>.api.getdal.sa/services/crm/clients/bulk",
apiKey: "your-api-key-here",
tenant: "your-tenant-name",
batchSize: 5000,
maxRetries: 3,
retryDelay: 3000,
};
const nationalityMap: Record<string, string> = {
سعودي: "SA",
باكستاني: "PK",
أردني: "JO",
مصري: "EG",
};
// Note: In a real-world scenario, client data would typically be fetched from a database rather than a CSV file
async function processCsvFile(filePath: string): Promise<void> {
const clients: Client[] = [];
return new Promise((resolve, reject) => {
createReadStream(filePath)
.pipe(csv())
.on("data", (row) => {
const client = mapToClient(row);
if (isValidClient(client)) {
clients.push(client);
}
})
.on("end", () => {
processBatches(clients).then(resolve).catch(reject);
})
.on("error", reject);
});
}
function mapToClient(row: any): Client {
const client: Client = {
type: row["entity type"],
nameEn: row["name"],
nameAr: row["name ar"],
identities: [
{
type: row["Id type"],
number: row["Id number"],
},
],
nationalityCode: nationalityMap[row["Nationality"]?.trim()],
dobGregorian: toIsoDate(row["Date of birth"]),
};
return client;
}
function toIsoDate(dateStr: string): string | undefined {
if (!dateStr?.trim()) return undefined;
try {
return new Date(dateStr.trim()).toISOString().split("T")[0];
} catch {
return undefined;
}
}
function isValidClient(client: Client): boolean {
const required = ["type", "identities", "nameEn", "nameAr"];
return required.every((field) =>
client[field as keyof Client]?.toString().trim(),
);
}
async function sendBatch(clients: Client[]): Promise<boolean> {
const payload = {
clients,
duplicateMode: "Skip" as const,
};
for (let attempt = 1; attempt <= config.maxRetries; attempt++) {
try {
const response = await axios.post(config.apiEndpoint, payload, {
headers: {
"x-api-key": config.apiKey,
"x-tenant": config.tenant,
"Content-Type": "application/json",
},
timeout: 60000,
});
if (response.status === 201) {
return true;
}
} catch (error: any) {
console.error(`Attempt ${attempt} failed:`, error.message);
if (attempt < config.maxRetries) {
await new Promise((resolve) => setTimeout(resolve, config.retryDelay));
}
}
}
return false;
}
async function processBatches(clients: Client[]): Promise<void> {
let totalSent = 0;
let totalSkipped = 0;
let batchNumber = 0;
for (let i = 0; i < clients.length; i += config.batchSize) {
batchNumber++;
const batch = clients.slice(i, i + config.batchSize);
const validClients = batch.filter(isValidClient);
const skippedCount = batch.length - validClients.length;
totalSkipped += skippedCount;
if (skippedCount > 0) {
console.log(`Batch ${batchNumber}: Skipped ${skippedCount} invalid rows`);
}
if (validClients.length === 0) {
continue;
}
const success = await sendBatch(validClients);
if (success) {
totalSent += validClients.length;
}
console.log(
`Batch ${batchNumber} processed (${validClients.length} clients)`,
);
}
console.log(`Migration completed!`);
console.log(`Total clients sent: ${totalSent}`);
console.log(`Total skipped: ${totalSkipped}`);
}
// Usage
async function main() {
try {
await processCsvFile("datasets/clients.csv");
} catch (error) {
console.error("Migration failed:", error);
}
}
main();
