Python memory error Save split csv python

Question

I have all the data of a lead object from salesforce by python and I save it by csv. but since there is a lot of information I get python memory error .

**This code get python memory error code**

from simple_salesforce import Salesforce
from datetime import datetime
import csv
import os
import json
import account

SALESFORCE_USERNAME = '123'
PASSWORD = '123'
SECURITY_TOKEN = '123'

def main():
    # Authentication settings
    sf = Salesforce(username=SALESFORCE_USERNAME,
                    password=PASSWORD,
                    security_token=SECURITY_TOKEN)

    # Lead Column setting to be acquired
    columns = [
        "CreatedDate"
    ]
    sosl = 'SELECT {0[0]} FROM Lead'.format(
        columns)

    # Data acquisition with SOSL
    data = sf.query_all(sosl)

    # Delete CSV file if it exists
    output_csv = 'output.csv'
    if os.path.exists(output_csv):
        os.remove(output_csv)

    # Write to CSV file
    for k, v in data.items():
        if type(v) is list:
            with open(output_csv, 'w', newline="") as f:
                writer = csv.DictWriter(f, fieldnames=columns)
                writer.writeheader()
                for d in v:
                    data = json.loads(json.dumps(d))
                    del data['attributes']
                    writer.writerow(data)
if __name__ == '__main__':
    main()

That's why when there are more than 1000 lines in the csv I want csv are recorded as follows.

1 output1.csv (1000 row)
2 output2.csv (1000 row)
3 output3.csv ......

And I get the following error, what do I need to do so that I can get out this way? I want to split the cvs and I put in with open csv iterator = True, chunk size = 1000

Code

from simple_salesforce import Salesforce
from datetime import datetime
import csv
import os
import json
import account

SALESFORCE_USERNAME = '123'
PASSWORD = '123'
SECURITY_TOKEN = '123'

def main():
    # Authentication settings
    sf = Salesforce(username=SALESFORCE_USERNAME,
                    password=PASSWORD,
                    security_token=SECURITY_TOKEN)

    # Lead Column setting to be acquired
    columns = [
        "CreatedDate"
    ]
    sosl = 'SELECT {0[0]} FROM Lead'.format(
        columns)

    # Data acquisition with SOSL
    data = sf.query_all(sosl)

    # Delete CSV file if it exists
    output_csv = 'output.csv'
    if os.path.exists(output_csv):
        os.remove(output_csv)

    # Write to CSV file
    for k, v in data.items():
        if type(v) is list:
            with open(output_csv, 'w', newline="",iterator=True,chunksize=1000) as f:
                writer = csv.DictWriter(f, fieldnames=columns)
                writer.writeheader()
                for d in v:
                    data = json.loads(json.dumps(d))
                    del data['attributes']
                    writer.writerow(data)
if __name__ == '__main__':
    main()

Error message

Traceback (most recent call last):
  File "c:/Users/test/Documents/test/test5.py", line 44, in <module>
    main()
  File "c:/Users/test/Documents//test5.py", line 36, in main
    with open(output_csv, 'w', newline="",iterator=True,chunksize=1000) as f:
TypeError: 'iterator' is an invalid keyword argument for open()

With this way I think that I will not get python error if there is another way they can teach me?

If anyone knows, please let me know.

question from:https://stackoverflow.com/questions/65839009/python-memory-error-save-split-csv-python

Answer 1

data = sf.query_all(sosl)

This call retrieves all information into memory for the given query, which is SOQL, not SOSL.

Instead, use

data = sf.query_all_iter(sosl)

and iterate over the resulting iterator instead of data.items(), which will be much more memory-efficient as it won't attempt to retrieve all items at once.