Skip to main content
added 101 characters in body
Source Link
drk
  • 81
  • 4

Based on the helpful comments from the community, I was able to solve the problem by writing the file in chunks. Here is the code with the adjustable chunk size. It worked on a small laptop with 8GB of RAM, even though it took 45 minutes.

rows = 100000000; (*MaxMemoryUsed: 5GB, Expected CSV file size: 1.3GB*)
largeTable = Table[{k, 0, 0}, {k, rows}];

Clear[ExportLargeTableToCSV]; 
ExportLargeTableToCSV[largeFileName_String, largeTable_List, 
  maxRowsPerChunk_Integer : 1000000] := Block[{largeFile, maxRows, i},
  
  If[FileExistsQ[largeFileName], DeleteFile[largeFileName];];
  largeFile = OpenAppend[largeFileName];
  maxRows = Min[Length[largeTable], maxRowsPerChunk];
  Print[ToString[Length[largeTable]] ~~ 
    " rows in the table. Chunk size:" ~~ ToString[maxRows] ~~ 
    " rows."];
  For[
   i = maxRows1, i <= Length[largeTable], i += maxRows,
   PrintTemporary["SavingPrint["Saving chunk " ~~ ToString[i/ToString[Quotient[i, maxRows] + 1] ~~ 
     " of " ~~ 
     ToString[Quotient[Length[largeTable]ToString[
      Quotient[Length[largeTable], maxRows]]maxRows] + 
       If[Mod[Length[largeTable], maxRows] == 0, 0, 1]] ~~ "..."];
   Export[largeFile, largeTable[[i 
 - maxRows + 1largeTable[[i ;; i]]Min[i + maxRows, Length[largeTable]]]], "CSV"];
   ];
  Close[largeFileName];
  Print["Done."];
  ];

ExportLargeTableToCSV[NotebookDirectory[] ~~ "largeTableChunks.csv", largeTable];

Based on the helpful comments from the community, I was able to solve the problem by writing the file in chunks. Here is the code with the adjustable chunk size. It worked on a small laptop with 8GB of RAM, even though it took 45 minutes.

rows = 100000000; (*MaxMemoryUsed: 5GB, Expected CSV file size: 1.3GB*)
largeTable = Table[{k, 0, 0}, {k, rows}];

Clear[ExportLargeTableToCSV]; 
ExportLargeTableToCSV[largeFileName_String, largeTable_List, 
  maxRowsPerChunk_Integer : 1000000] := Block[{largeFile, maxRows, i},
  
  If[FileExistsQ[largeFileName], DeleteFile[largeFileName];];
  largeFile = OpenAppend[largeFileName];
  maxRows = Min[Length[largeTable], maxRowsPerChunk];
  Print[ToString[Length[largeTable]] ~~ 
    " rows in the table. Chunk size:" ~~ ToString[maxRows] ~~ 
    " rows."];
  For[
   i = maxRows, i <= Length[largeTable], i += maxRows,
   PrintTemporary["Saving chunk " ~~ ToString[i/maxRows] ~~ " of " ~~ 
     ToString[Quotient[Length[largeTable], maxRows]] ~~ "..."];
   Export[largeFile, largeTable[[i - maxRows + 1 ;; i]], "CSV"];
   ];
  Close[largeFileName];
  Print["Done."];
  ];

ExportLargeTableToCSV[NotebookDirectory[] ~~ "largeTableChunks.csv", largeTable];

Based on the helpful comments from the community, I was able to solve the problem by writing the file in chunks. Here is the code with the adjustable chunk size. It worked on a small laptop with 8GB of RAM, even though it took 45 minutes.

rows = 100000000; (*MaxMemoryUsed: 5GB, Expected CSV file size: 1.3GB*)
largeTable = Table[{k, 0, 0}, {k, rows}];

Clear[ExportLargeTableToCSV]; 
ExportLargeTableToCSV[largeFileName_String, largeTable_List, 
  maxRowsPerChunk_Integer : 1000000] := Block[{largeFile, maxRows, i},
  
  If[FileExistsQ[largeFileName], DeleteFile[largeFileName];];
  largeFile = OpenAppend[largeFileName];
  maxRows = Min[Length[largeTable], maxRowsPerChunk];
  Print[ToString[Length[largeTable]] ~~ 
    " rows in the table. Chunk size:" ~~ ToString[maxRows] ~~ 
    " rows."];
  For[
   i = 1, i <= Length[largeTable], i += maxRows,
   Print["Saving chunk " ~~ ToString[Quotient[i, maxRows] + 1] ~~ 
     " of " ~~ 
     ToString[
      Quotient[Length[largeTable], maxRows] + 
       If[Mod[Length[largeTable], maxRows] == 0, 0, 1]] ~~ "..."];
   Export[largeFile,  
    largeTable[[i ;; Min[i + maxRows, Length[largeTable]]]], "CSV"];
   ];
  Close[largeFileName];
  Print["Done."];
  ];

ExportLargeTableToCSV[NotebookDirectory[] ~~ "largeTableChunks.csv", largeTable];
Source Link
drk
  • 81
  • 4

Based on the helpful comments from the community, I was able to solve the problem by writing the file in chunks. Here is the code with the adjustable chunk size. It worked on a small laptop with 8GB of RAM, even though it took 45 minutes.

rows = 100000000; (*MaxMemoryUsed: 5GB, Expected CSV file size: 1.3GB*)
largeTable = Table[{k, 0, 0}, {k, rows}];

Clear[ExportLargeTableToCSV]; 
ExportLargeTableToCSV[largeFileName_String, largeTable_List, 
  maxRowsPerChunk_Integer : 1000000] := Block[{largeFile, maxRows, i},
  
  If[FileExistsQ[largeFileName], DeleteFile[largeFileName];];
  largeFile = OpenAppend[largeFileName];
  maxRows = Min[Length[largeTable], maxRowsPerChunk];
  Print[ToString[Length[largeTable]] ~~ 
    " rows in the table. Chunk size:" ~~ ToString[maxRows] ~~ 
    " rows."];
  For[
   i = maxRows, i <= Length[largeTable], i += maxRows,
   PrintTemporary["Saving chunk " ~~ ToString[i/maxRows] ~~ " of " ~~ 
     ToString[Quotient[Length[largeTable], maxRows]] ~~ "..."];
   Export[largeFile, largeTable[[i - maxRows + 1 ;; i]], "CSV"];
   ];
  Close[largeFileName];
  Print["Done."];
  ];

ExportLargeTableToCSV[NotebookDirectory[] ~~ "largeTableChunks.csv", largeTable];