sync.cjs 52 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334
  1. 'use strict';
  2. class CsvError extends Error {
  3. constructor(code, message, options, ...contexts) {
  4. if(Array.isArray(message)) message = message.join(' ').trim();
  5. super(message);
  6. if(Error.captureStackTrace !== undefined){
  7. Error.captureStackTrace(this, CsvError);
  8. }
  9. this.code = code;
  10. for(const context of contexts){
  11. for(const key in context){
  12. const value = context[key];
  13. this[key] = Buffer.isBuffer(value) ? value.toString(options.encoding) : value == null ? value : JSON.parse(JSON.stringify(value));
  14. }
  15. }
  16. }
  17. }
  18. const is_object = function(obj){
  19. return (typeof obj === 'object' && obj !== null && !Array.isArray(obj));
  20. };
  21. const normalize_columns_array = function(columns){
  22. const normalizedColumns = [];
  23. for(let i = 0, l = columns.length; i < l; i++){
  24. const column = columns[i];
  25. if(column === undefined || column === null || column === false){
  26. normalizedColumns[i] = { disabled: true };
  27. }else if(typeof column === 'string'){
  28. normalizedColumns[i] = { name: column };
  29. }else if(is_object(column)){
  30. if(typeof column.name !== 'string'){
  31. throw new CsvError('CSV_OPTION_COLUMNS_MISSING_NAME', [
  32. 'Option columns missing name:',
  33. `property "name" is required at position ${i}`,
  34. 'when column is an object literal'
  35. ]);
  36. }
  37. normalizedColumns[i] = column;
  38. }else {
  39. throw new CsvError('CSV_INVALID_COLUMN_DEFINITION', [
  40. 'Invalid column definition:',
  41. 'expect a string or a literal object,',
  42. `got ${JSON.stringify(column)} at position ${i}`
  43. ]);
  44. }
  45. }
  46. return normalizedColumns;
  47. };
  48. class ResizeableBuffer{
  49. constructor(size=100){
  50. this.size = size;
  51. this.length = 0;
  52. this.buf = Buffer.allocUnsafe(size);
  53. }
  54. prepend(val){
  55. if(Buffer.isBuffer(val)){
  56. const length = this.length + val.length;
  57. if(length >= this.size){
  58. this.resize();
  59. if(length >= this.size){
  60. throw Error('INVALID_BUFFER_STATE');
  61. }
  62. }
  63. const buf = this.buf;
  64. this.buf = Buffer.allocUnsafe(this.size);
  65. val.copy(this.buf, 0);
  66. buf.copy(this.buf, val.length);
  67. this.length += val.length;
  68. }else {
  69. const length = this.length++;
  70. if(length === this.size){
  71. this.resize();
  72. }
  73. const buf = this.clone();
  74. this.buf[0] = val;
  75. buf.copy(this.buf,1, 0, length);
  76. }
  77. }
  78. append(val){
  79. const length = this.length++;
  80. if(length === this.size){
  81. this.resize();
  82. }
  83. this.buf[length] = val;
  84. }
  85. clone(){
  86. return Buffer.from(this.buf.slice(0, this.length));
  87. }
  88. resize(){
  89. const length = this.length;
  90. this.size = this.size * 2;
  91. const buf = Buffer.allocUnsafe(this.size);
  92. this.buf.copy(buf,0, 0, length);
  93. this.buf = buf;
  94. }
  95. toString(encoding){
  96. if(encoding){
  97. return this.buf.slice(0, this.length).toString(encoding);
  98. }else {
  99. return Uint8Array.prototype.slice.call(this.buf.slice(0, this.length));
  100. }
  101. }
  102. toJSON(){
  103. return this.toString('utf8');
  104. }
  105. reset(){
  106. this.length = 0;
  107. }
  108. }
  109. // white space characters
  110. // https://en.wikipedia.org/wiki/Whitespace_character
  111. // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions/Character_Classes#Types
  112. // \f\n\r\t\v\u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff
  113. const np = 12;
  114. const cr$1 = 13; // `\r`, carriage return, 0x0D in hexadécimal, 13 in decimal
  115. const nl$1 = 10; // `\n`, newline, 0x0A in hexadecimal, 10 in decimal
  116. const space = 32;
  117. const tab = 9;
  118. const init_state = function(options){
  119. return {
  120. bomSkipped: false,
  121. bufBytesStart: 0,
  122. castField: options.cast_function,
  123. commenting: false,
  124. // Current error encountered by a record
  125. error: undefined,
  126. enabled: options.from_line === 1,
  127. escaping: false,
  128. escapeIsQuote: Buffer.isBuffer(options.escape) && Buffer.isBuffer(options.quote) && Buffer.compare(options.escape, options.quote) === 0,
  129. // columns can be `false`, `true`, `Array`
  130. expectedRecordLength: Array.isArray(options.columns) ? options.columns.length : undefined,
  131. field: new ResizeableBuffer(20),
  132. firstLineToHeaders: options.cast_first_line_to_header,
  133. needMoreDataSize: Math.max(
  134. // Skip if the remaining buffer smaller than comment
  135. options.comment !== null ? options.comment.length : 0,
  136. // Skip if the remaining buffer can be delimiter
  137. ...options.delimiter.map((delimiter) => delimiter.length),
  138. // Skip if the remaining buffer can be escape sequence
  139. options.quote !== null ? options.quote.length : 0,
  140. ),
  141. previousBuf: undefined,
  142. quoting: false,
  143. stop: false,
  144. rawBuffer: new ResizeableBuffer(100),
  145. record: [],
  146. recordHasError: false,
  147. record_length: 0,
  148. recordDelimiterMaxLength: options.record_delimiter.length === 0 ? 0 : Math.max(...options.record_delimiter.map((v) => v.length)),
  149. trimChars: [Buffer.from(' ', options.encoding)[0], Buffer.from('\t', options.encoding)[0]],
  150. wasQuoting: false,
  151. wasRowDelimiter: false,
  152. timchars: [
  153. Buffer.from(Buffer.from([cr$1], 'utf8').toString(), options.encoding),
  154. Buffer.from(Buffer.from([nl$1], 'utf8').toString(), options.encoding),
  155. Buffer.from(Buffer.from([np], 'utf8').toString(), options.encoding),
  156. Buffer.from(Buffer.from([space], 'utf8').toString(), options.encoding),
  157. Buffer.from(Buffer.from([tab], 'utf8').toString(), options.encoding),
  158. ]
  159. };
  160. };
  161. const underscore = function(str){
  162. return str.replace(/([A-Z])/g, function(_, match){
  163. return '_' + match.toLowerCase();
  164. });
  165. };
  166. const normalize_options = function(opts){
  167. const options = {};
  168. // Merge with user options
  169. for(const opt in opts){
  170. options[underscore(opt)] = opts[opt];
  171. }
  172. // Normalize option `encoding`
  173. // Note: defined first because other options depends on it
  174. // to convert chars/strings into buffers.
  175. if(options.encoding === undefined || options.encoding === true){
  176. options.encoding = 'utf8';
  177. }else if(options.encoding === null || options.encoding === false){
  178. options.encoding = null;
  179. }else if(typeof options.encoding !== 'string' && options.encoding !== null){
  180. throw new CsvError('CSV_INVALID_OPTION_ENCODING', [
  181. 'Invalid option encoding:',
  182. 'encoding must be a string or null to return a buffer,',
  183. `got ${JSON.stringify(options.encoding)}`
  184. ], options);
  185. }
  186. // Normalize option `bom`
  187. if(options.bom === undefined || options.bom === null || options.bom === false){
  188. options.bom = false;
  189. }else if(options.bom !== true){
  190. throw new CsvError('CSV_INVALID_OPTION_BOM', [
  191. 'Invalid option bom:', 'bom must be true,',
  192. `got ${JSON.stringify(options.bom)}`
  193. ], options);
  194. }
  195. // Normalize option `cast`
  196. options.cast_function = null;
  197. if(options.cast === undefined || options.cast === null || options.cast === false || options.cast === ''){
  198. options.cast = undefined;
  199. }else if(typeof options.cast === 'function'){
  200. options.cast_function = options.cast;
  201. options.cast = true;
  202. }else if(options.cast !== true){
  203. throw new CsvError('CSV_INVALID_OPTION_CAST', [
  204. 'Invalid option cast:', 'cast must be true or a function,',
  205. `got ${JSON.stringify(options.cast)}`
  206. ], options);
  207. }
  208. // Normalize option `cast_date`
  209. if(options.cast_date === undefined || options.cast_date === null || options.cast_date === false || options.cast_date === ''){
  210. options.cast_date = false;
  211. }else if(options.cast_date === true){
  212. options.cast_date = function(value){
  213. const date = Date.parse(value);
  214. return !isNaN(date) ? new Date(date) : value;
  215. };
  216. }else if (typeof options.cast_date !== 'function'){
  217. throw new CsvError('CSV_INVALID_OPTION_CAST_DATE', [
  218. 'Invalid option cast_date:', 'cast_date must be true or a function,',
  219. `got ${JSON.stringify(options.cast_date)}`
  220. ], options);
  221. }
  222. // Normalize option `columns`
  223. options.cast_first_line_to_header = null;
  224. if(options.columns === true){
  225. // Fields in the first line are converted as-is to columns
  226. options.cast_first_line_to_header = undefined;
  227. }else if(typeof options.columns === 'function'){
  228. options.cast_first_line_to_header = options.columns;
  229. options.columns = true;
  230. }else if(Array.isArray(options.columns)){
  231. options.columns = normalize_columns_array(options.columns);
  232. }else if(options.columns === undefined || options.columns === null || options.columns === false){
  233. options.columns = false;
  234. }else {
  235. throw new CsvError('CSV_INVALID_OPTION_COLUMNS', [
  236. 'Invalid option columns:',
  237. 'expect an array, a function or true,',
  238. `got ${JSON.stringify(options.columns)}`
  239. ], options);
  240. }
  241. // Normalize option `group_columns_by_name`
  242. if(options.group_columns_by_name === undefined || options.group_columns_by_name === null || options.group_columns_by_name === false){
  243. options.group_columns_by_name = false;
  244. }else if(options.group_columns_by_name !== true){
  245. throw new CsvError('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [
  246. 'Invalid option group_columns_by_name:',
  247. 'expect an boolean,',
  248. `got ${JSON.stringify(options.group_columns_by_name)}`
  249. ], options);
  250. }else if(options.columns === false){
  251. throw new CsvError('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [
  252. 'Invalid option group_columns_by_name:',
  253. 'the `columns` mode must be activated.'
  254. ], options);
  255. }
  256. // Normalize option `comment`
  257. if(options.comment === undefined || options.comment === null || options.comment === false || options.comment === ''){
  258. options.comment = null;
  259. }else {
  260. if(typeof options.comment === 'string'){
  261. options.comment = Buffer.from(options.comment, options.encoding);
  262. }
  263. if(!Buffer.isBuffer(options.comment)){
  264. throw new CsvError('CSV_INVALID_OPTION_COMMENT', [
  265. 'Invalid option comment:',
  266. 'comment must be a buffer or a string,',
  267. `got ${JSON.stringify(options.comment)}`
  268. ], options);
  269. }
  270. }
  271. // Normalize option `comment_no_infix`
  272. if(options.comment_no_infix === undefined || options.comment_no_infix === null || options.comment_no_infix === false){
  273. options.comment_no_infix = false;
  274. }else if(options.comment_no_infix !== true){
  275. throw new CsvError('CSV_INVALID_OPTION_COMMENT', [
  276. 'Invalid option comment_no_infix:',
  277. 'value must be a boolean,',
  278. `got ${JSON.stringify(options.comment_no_infix)}`
  279. ], options);
  280. }
  281. // Normalize option `delimiter`
  282. const delimiter_json = JSON.stringify(options.delimiter);
  283. if(!Array.isArray(options.delimiter)) options.delimiter = [options.delimiter];
  284. if(options.delimiter.length === 0){
  285. throw new CsvError('CSV_INVALID_OPTION_DELIMITER', [
  286. 'Invalid option delimiter:',
  287. 'delimiter must be a non empty string or buffer or array of string|buffer,',
  288. `got ${delimiter_json}`
  289. ], options);
  290. }
  291. options.delimiter = options.delimiter.map(function(delimiter){
  292. if(delimiter === undefined || delimiter === null || delimiter === false){
  293. return Buffer.from(',', options.encoding);
  294. }
  295. if(typeof delimiter === 'string'){
  296. delimiter = Buffer.from(delimiter, options.encoding);
  297. }
  298. if(!Buffer.isBuffer(delimiter) || delimiter.length === 0){
  299. throw new CsvError('CSV_INVALID_OPTION_DELIMITER', [
  300. 'Invalid option delimiter:',
  301. 'delimiter must be a non empty string or buffer or array of string|buffer,',
  302. `got ${delimiter_json}`
  303. ], options);
  304. }
  305. return delimiter;
  306. });
  307. // Normalize option `escape`
  308. if(options.escape === undefined || options.escape === true){
  309. options.escape = Buffer.from('"', options.encoding);
  310. }else if(typeof options.escape === 'string'){
  311. options.escape = Buffer.from(options.escape, options.encoding);
  312. }else if (options.escape === null || options.escape === false){
  313. options.escape = null;
  314. }
  315. if(options.escape !== null){
  316. if(!Buffer.isBuffer(options.escape)){
  317. throw new Error(`Invalid Option: escape must be a buffer, a string or a boolean, got ${JSON.stringify(options.escape)}`);
  318. }
  319. }
  320. // Normalize option `from`
  321. if(options.from === undefined || options.from === null){
  322. options.from = 1;
  323. }else {
  324. if(typeof options.from === 'string' && /\d+/.test(options.from)){
  325. options.from = parseInt(options.from);
  326. }
  327. if(Number.isInteger(options.from)){
  328. if(options.from < 0){
  329. throw new Error(`Invalid Option: from must be a positive integer, got ${JSON.stringify(opts.from)}`);
  330. }
  331. }else {
  332. throw new Error(`Invalid Option: from must be an integer, got ${JSON.stringify(options.from)}`);
  333. }
  334. }
  335. // Normalize option `from_line`
  336. if(options.from_line === undefined || options.from_line === null){
  337. options.from_line = 1;
  338. }else {
  339. if(typeof options.from_line === 'string' && /\d+/.test(options.from_line)){
  340. options.from_line = parseInt(options.from_line);
  341. }
  342. if(Number.isInteger(options.from_line)){
  343. if(options.from_line <= 0){
  344. throw new Error(`Invalid Option: from_line must be a positive integer greater than 0, got ${JSON.stringify(opts.from_line)}`);
  345. }
  346. }else {
  347. throw new Error(`Invalid Option: from_line must be an integer, got ${JSON.stringify(opts.from_line)}`);
  348. }
  349. }
  350. // Normalize options `ignore_last_delimiters`
  351. if(options.ignore_last_delimiters === undefined || options.ignore_last_delimiters === null){
  352. options.ignore_last_delimiters = false;
  353. }else if(typeof options.ignore_last_delimiters === 'number'){
  354. options.ignore_last_delimiters = Math.floor(options.ignore_last_delimiters);
  355. if(options.ignore_last_delimiters === 0){
  356. options.ignore_last_delimiters = false;
  357. }
  358. }else if(typeof options.ignore_last_delimiters !== 'boolean'){
  359. throw new CsvError('CSV_INVALID_OPTION_IGNORE_LAST_DELIMITERS', [
  360. 'Invalid option `ignore_last_delimiters`:',
  361. 'the value must be a boolean value or an integer,',
  362. `got ${JSON.stringify(options.ignore_last_delimiters)}`
  363. ], options);
  364. }
  365. if(options.ignore_last_delimiters === true && options.columns === false){
  366. throw new CsvError('CSV_IGNORE_LAST_DELIMITERS_REQUIRES_COLUMNS', [
  367. 'The option `ignore_last_delimiters`',
  368. 'requires the activation of the `columns` option'
  369. ], options);
  370. }
  371. // Normalize option `info`
  372. if(options.info === undefined || options.info === null || options.info === false){
  373. options.info = false;
  374. }else if(options.info !== true){
  375. throw new Error(`Invalid Option: info must be true, got ${JSON.stringify(options.info)}`);
  376. }
  377. // Normalize option `max_record_size`
  378. if(options.max_record_size === undefined || options.max_record_size === null || options.max_record_size === false){
  379. options.max_record_size = 0;
  380. }else if(Number.isInteger(options.max_record_size) && options.max_record_size >= 0);else if(typeof options.max_record_size === 'string' && /\d+/.test(options.max_record_size)){
  381. options.max_record_size = parseInt(options.max_record_size);
  382. }else {
  383. throw new Error(`Invalid Option: max_record_size must be a positive integer, got ${JSON.stringify(options.max_record_size)}`);
  384. }
  385. // Normalize option `objname`
  386. if(options.objname === undefined || options.objname === null || options.objname === false){
  387. options.objname = undefined;
  388. }else if(Buffer.isBuffer(options.objname)){
  389. if(options.objname.length === 0){
  390. throw new Error(`Invalid Option: objname must be a non empty buffer`);
  391. }
  392. if(options.encoding === null);else {
  393. options.objname = options.objname.toString(options.encoding);
  394. }
  395. }else if(typeof options.objname === 'string'){
  396. if(options.objname.length === 0){
  397. throw new Error(`Invalid Option: objname must be a non empty string`);
  398. }
  399. // Great, nothing to do
  400. }else if(typeof options.objname === 'number');else {
  401. throw new Error(`Invalid Option: objname must be a string or a buffer, got ${options.objname}`);
  402. }
  403. if(options.objname !== undefined){
  404. if(typeof options.objname === 'number'){
  405. if(options.columns !== false){
  406. throw Error('Invalid Option: objname index cannot be combined with columns or be defined as a field');
  407. }
  408. }else { // A string or a buffer
  409. if(options.columns === false){
  410. throw Error('Invalid Option: objname field must be combined with columns or be defined as an index');
  411. }
  412. }
  413. }
  414. // Normalize option `on_record`
  415. if(options.on_record === undefined || options.on_record === null){
  416. options.on_record = undefined;
  417. }else if(typeof options.on_record !== 'function'){
  418. throw new CsvError('CSV_INVALID_OPTION_ON_RECORD', [
  419. 'Invalid option `on_record`:',
  420. 'expect a function,',
  421. `got ${JSON.stringify(options.on_record)}`
  422. ], options);
  423. }
  424. // Normalize option `quote`
  425. if(options.quote === null || options.quote === false || options.quote === ''){
  426. options.quote = null;
  427. }else {
  428. if(options.quote === undefined || options.quote === true){
  429. options.quote = Buffer.from('"', options.encoding);
  430. }else if(typeof options.quote === 'string'){
  431. options.quote = Buffer.from(options.quote, options.encoding);
  432. }
  433. if(!Buffer.isBuffer(options.quote)){
  434. throw new Error(`Invalid Option: quote must be a buffer or a string, got ${JSON.stringify(options.quote)}`);
  435. }
  436. }
  437. // Normalize option `raw`
  438. if(options.raw === undefined || options.raw === null || options.raw === false){
  439. options.raw = false;
  440. }else if(options.raw !== true){
  441. throw new Error(`Invalid Option: raw must be true, got ${JSON.stringify(options.raw)}`);
  442. }
  443. // Normalize option `record_delimiter`
  444. if(options.record_delimiter === undefined){
  445. options.record_delimiter = [];
  446. }else if(typeof options.record_delimiter === 'string' || Buffer.isBuffer(options.record_delimiter)){
  447. if(options.record_delimiter.length === 0){
  448. throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [
  449. 'Invalid option `record_delimiter`:',
  450. 'value must be a non empty string or buffer,',
  451. `got ${JSON.stringify(options.record_delimiter)}`
  452. ], options);
  453. }
  454. options.record_delimiter = [options.record_delimiter];
  455. }else if(!Array.isArray(options.record_delimiter)){
  456. throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [
  457. 'Invalid option `record_delimiter`:',
  458. 'value must be a string, a buffer or array of string|buffer,',
  459. `got ${JSON.stringify(options.record_delimiter)}`
  460. ], options);
  461. }
  462. options.record_delimiter = options.record_delimiter.map(function(rd, i){
  463. if(typeof rd !== 'string' && ! Buffer.isBuffer(rd)){
  464. throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [
  465. 'Invalid option `record_delimiter`:',
  466. 'value must be a string, a buffer or array of string|buffer',
  467. `at index ${i},`,
  468. `got ${JSON.stringify(rd)}`
  469. ], options);
  470. }else if(rd.length === 0){
  471. throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [
  472. 'Invalid option `record_delimiter`:',
  473. 'value must be a non empty string or buffer',
  474. `at index ${i},`,
  475. `got ${JSON.stringify(rd)}`
  476. ], options);
  477. }
  478. if(typeof rd === 'string'){
  479. rd = Buffer.from(rd, options.encoding);
  480. }
  481. return rd;
  482. });
  483. // Normalize option `relax_column_count`
  484. if(typeof options.relax_column_count === 'boolean');else if(options.relax_column_count === undefined || options.relax_column_count === null){
  485. options.relax_column_count = false;
  486. }else {
  487. throw new Error(`Invalid Option: relax_column_count must be a boolean, got ${JSON.stringify(options.relax_column_count)}`);
  488. }
  489. if(typeof options.relax_column_count_less === 'boolean');else if(options.relax_column_count_less === undefined || options.relax_column_count_less === null){
  490. options.relax_column_count_less = false;
  491. }else {
  492. throw new Error(`Invalid Option: relax_column_count_less must be a boolean, got ${JSON.stringify(options.relax_column_count_less)}`);
  493. }
  494. if(typeof options.relax_column_count_more === 'boolean');else if(options.relax_column_count_more === undefined || options.relax_column_count_more === null){
  495. options.relax_column_count_more = false;
  496. }else {
  497. throw new Error(`Invalid Option: relax_column_count_more must be a boolean, got ${JSON.stringify(options.relax_column_count_more)}`);
  498. }
  499. // Normalize option `relax_quotes`
  500. if(typeof options.relax_quotes === 'boolean');else if(options.relax_quotes === undefined || options.relax_quotes === null){
  501. options.relax_quotes = false;
  502. }else {
  503. throw new Error(`Invalid Option: relax_quotes must be a boolean, got ${JSON.stringify(options.relax_quotes)}`);
  504. }
  505. // Normalize option `skip_empty_lines`
  506. if(typeof options.skip_empty_lines === 'boolean');else if(options.skip_empty_lines === undefined || options.skip_empty_lines === null){
  507. options.skip_empty_lines = false;
  508. }else {
  509. throw new Error(`Invalid Option: skip_empty_lines must be a boolean, got ${JSON.stringify(options.skip_empty_lines)}`);
  510. }
  511. // Normalize option `skip_records_with_empty_values`
  512. if(typeof options.skip_records_with_empty_values === 'boolean');else if(options.skip_records_with_empty_values === undefined || options.skip_records_with_empty_values === null){
  513. options.skip_records_with_empty_values = false;
  514. }else {
  515. throw new Error(`Invalid Option: skip_records_with_empty_values must be a boolean, got ${JSON.stringify(options.skip_records_with_empty_values)}`);
  516. }
  517. // Normalize option `skip_records_with_error`
  518. if(typeof options.skip_records_with_error === 'boolean');else if(options.skip_records_with_error === undefined || options.skip_records_with_error === null){
  519. options.skip_records_with_error = false;
  520. }else {
  521. throw new Error(`Invalid Option: skip_records_with_error must be a boolean, got ${JSON.stringify(options.skip_records_with_error)}`);
  522. }
  523. // Normalize option `rtrim`
  524. if(options.rtrim === undefined || options.rtrim === null || options.rtrim === false){
  525. options.rtrim = false;
  526. }else if(options.rtrim !== true){
  527. throw new Error(`Invalid Option: rtrim must be a boolean, got ${JSON.stringify(options.rtrim)}`);
  528. }
  529. // Normalize option `ltrim`
  530. if(options.ltrim === undefined || options.ltrim === null || options.ltrim === false){
  531. options.ltrim = false;
  532. }else if(options.ltrim !== true){
  533. throw new Error(`Invalid Option: ltrim must be a boolean, got ${JSON.stringify(options.ltrim)}`);
  534. }
  535. // Normalize option `trim`
  536. if(options.trim === undefined || options.trim === null || options.trim === false){
  537. options.trim = false;
  538. }else if(options.trim !== true){
  539. throw new Error(`Invalid Option: trim must be a boolean, got ${JSON.stringify(options.trim)}`);
  540. }
  541. // Normalize options `trim`, `ltrim` and `rtrim`
  542. if(options.trim === true && opts.ltrim !== false){
  543. options.ltrim = true;
  544. }else if(options.ltrim !== true){
  545. options.ltrim = false;
  546. }
  547. if(options.trim === true && opts.rtrim !== false){
  548. options.rtrim = true;
  549. }else if(options.rtrim !== true){
  550. options.rtrim = false;
  551. }
  552. // Normalize option `to`
  553. if(options.to === undefined || options.to === null){
  554. options.to = -1;
  555. }else {
  556. if(typeof options.to === 'string' && /\d+/.test(options.to)){
  557. options.to = parseInt(options.to);
  558. }
  559. if(Number.isInteger(options.to)){
  560. if(options.to <= 0){
  561. throw new Error(`Invalid Option: to must be a positive integer greater than 0, got ${JSON.stringify(opts.to)}`);
  562. }
  563. }else {
  564. throw new Error(`Invalid Option: to must be an integer, got ${JSON.stringify(opts.to)}`);
  565. }
  566. }
  567. // Normalize option `to_line`
  568. if(options.to_line === undefined || options.to_line === null){
  569. options.to_line = -1;
  570. }else {
  571. if(typeof options.to_line === 'string' && /\d+/.test(options.to_line)){
  572. options.to_line = parseInt(options.to_line);
  573. }
  574. if(Number.isInteger(options.to_line)){
  575. if(options.to_line <= 0){
  576. throw new Error(`Invalid Option: to_line must be a positive integer greater than 0, got ${JSON.stringify(opts.to_line)}`);
  577. }
  578. }else {
  579. throw new Error(`Invalid Option: to_line must be an integer, got ${JSON.stringify(opts.to_line)}`);
  580. }
  581. }
  582. return options;
  583. };
  584. const isRecordEmpty = function(record){
  585. return record.every((field) => field == null || field.toString && field.toString().trim() === '');
  586. };
  587. const cr = 13; // `\r`, carriage return, 0x0D in hexadécimal, 13 in decimal
  588. const nl = 10; // `\n`, newline, 0x0A in hexadecimal, 10 in decimal
  589. const boms = {
  590. // Note, the following are equals:
  591. // Buffer.from("\ufeff")
  592. // Buffer.from([239, 187, 191])
  593. // Buffer.from('EFBBBF', 'hex')
  594. 'utf8': Buffer.from([239, 187, 191]),
  595. // Note, the following are equals:
  596. // Buffer.from "\ufeff", 'utf16le
  597. // Buffer.from([255, 254])
  598. 'utf16le': Buffer.from([255, 254])
  599. };
  600. const transform = function(original_options = {}) {
  601. const info = {
  602. bytes: 0,
  603. comment_lines: 0,
  604. empty_lines: 0,
  605. invalid_field_length: 0,
  606. lines: 1,
  607. records: 0
  608. };
  609. const options = normalize_options(original_options);
  610. return {
  611. info: info,
  612. original_options: original_options,
  613. options: options,
  614. state: init_state(options),
  615. __needMoreData: function(i, bufLen, end){
  616. if(end) return false;
  617. const {encoding, escape, quote} = this.options;
  618. const {quoting, needMoreDataSize, recordDelimiterMaxLength} = this.state;
  619. const numOfCharLeft = bufLen - i - 1;
  620. const requiredLength = Math.max(
  621. needMoreDataSize,
  622. // Skip if the remaining buffer smaller than record delimiter
  623. // If "record_delimiter" is yet to be discovered:
  624. // 1. It is equals to `[]` and "recordDelimiterMaxLength" equals `0`
  625. // 2. We set the length to windows line ending in the current encoding
  626. // Note, that encoding is known from user or bom discovery at that point
  627. // recordDelimiterMaxLength,
  628. recordDelimiterMaxLength === 0 ? Buffer.from('\r\n', encoding).length : recordDelimiterMaxLength,
  629. // Skip if remaining buffer can be an escaped quote
  630. quoting ? ((escape === null ? 0 : escape.length) + quote.length) : 0,
  631. // Skip if remaining buffer can be record delimiter following the closing quote
  632. quoting ? (quote.length + recordDelimiterMaxLength) : 0,
  633. );
  634. return numOfCharLeft < requiredLength;
  635. },
  636. // Central parser implementation
  637. parse: function(nextBuf, end, push, close){
  638. const {bom, comment_no_infix, encoding, from_line, ltrim, max_record_size,raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options;
  639. let {comment, escape, quote, record_delimiter} = this.options;
  640. const {bomSkipped, previousBuf, rawBuffer, escapeIsQuote} = this.state;
  641. let buf;
  642. if(previousBuf === undefined){
  643. if(nextBuf === undefined){
  644. // Handle empty string
  645. close();
  646. return;
  647. }else {
  648. buf = nextBuf;
  649. }
  650. }else if(previousBuf !== undefined && nextBuf === undefined){
  651. buf = previousBuf;
  652. }else {
  653. buf = Buffer.concat([previousBuf, nextBuf]);
  654. }
  655. // Handle UTF BOM
  656. if(bomSkipped === false){
  657. if(bom === false){
  658. this.state.bomSkipped = true;
  659. }else if(buf.length < 3){
  660. // No enough data
  661. if(end === false){
  662. // Wait for more data
  663. this.state.previousBuf = buf;
  664. return;
  665. }
  666. }else {
  667. for(const encoding in boms){
  668. if(boms[encoding].compare(buf, 0, boms[encoding].length) === 0){
  669. // Skip BOM
  670. const bomLength = boms[encoding].length;
  671. this.state.bufBytesStart += bomLength;
  672. buf = buf.slice(bomLength);
  673. // Renormalize original options with the new encoding
  674. this.options = normalize_options({...this.original_options, encoding: encoding});
  675. // Options will re-evaluate the Buffer with the new encoding
  676. ({comment, escape, quote } = this.options);
  677. break;
  678. }
  679. }
  680. this.state.bomSkipped = true;
  681. }
  682. }
  683. const bufLen = buf.length;
  684. let pos;
  685. for(pos = 0; pos < bufLen; pos++){
  686. // Ensure we get enough space to look ahead
  687. // There should be a way to move this out of the loop
  688. if(this.__needMoreData(pos, bufLen, end)){
  689. break;
  690. }
  691. if(this.state.wasRowDelimiter === true){
  692. this.info.lines++;
  693. this.state.wasRowDelimiter = false;
  694. }
  695. if(to_line !== -1 && this.info.lines > to_line){
  696. this.state.stop = true;
  697. close();
  698. return;
  699. }
  700. // Auto discovery of record_delimiter, unix, mac and windows supported
  701. if(this.state.quoting === false && record_delimiter.length === 0){
  702. const record_delimiterCount = this.__autoDiscoverRecordDelimiter(buf, pos);
  703. if(record_delimiterCount){
  704. record_delimiter = this.options.record_delimiter;
  705. }
  706. }
  707. const chr = buf[pos];
  708. if(raw === true){
  709. rawBuffer.append(chr);
  710. }
  711. if((chr === cr || chr === nl) && this.state.wasRowDelimiter === false){
  712. this.state.wasRowDelimiter = true;
  713. }
  714. // Previous char was a valid escape char
  715. // treat the current char as a regular char
  716. if(this.state.escaping === true){
  717. this.state.escaping = false;
  718. }else {
  719. // Escape is only active inside quoted fields
  720. // We are quoting, the char is an escape chr and there is a chr to escape
  721. // if(escape !== null && this.state.quoting === true && chr === escape && pos + 1 < bufLen){
  722. if(escape !== null && this.state.quoting === true && this.__isEscape(buf, pos, chr) && pos + escape.length < bufLen){
  723. if(escapeIsQuote){
  724. if(this.__isQuote(buf, pos+escape.length)){
  725. this.state.escaping = true;
  726. pos += escape.length - 1;
  727. continue;
  728. }
  729. }else {
  730. this.state.escaping = true;
  731. pos += escape.length - 1;
  732. continue;
  733. }
  734. }
  735. // Not currently escaping and chr is a quote
  736. // TODO: need to compare bytes instead of single char
  737. if(this.state.commenting === false && this.__isQuote(buf, pos)){
  738. if(this.state.quoting === true){
  739. const nextChr = buf[pos+quote.length];
  740. const isNextChrTrimable = rtrim && this.__isCharTrimable(buf, pos+quote.length);
  741. const isNextChrComment = comment !== null && this.__compareBytes(comment, buf, pos+quote.length, nextChr);
  742. const isNextChrDelimiter = this.__isDelimiter(buf, pos+quote.length, nextChr);
  743. const isNextChrRecordDelimiter = record_delimiter.length === 0 ? this.__autoDiscoverRecordDelimiter(buf, pos+quote.length) : this.__isRecordDelimiter(nextChr, buf, pos+quote.length);
  744. // Escape a quote
  745. // Treat next char as a regular character
  746. if(escape !== null && this.__isEscape(buf, pos, chr) && this.__isQuote(buf, pos + escape.length)){
  747. pos += escape.length - 1;
  748. }else if(!nextChr || isNextChrDelimiter || isNextChrRecordDelimiter || isNextChrComment || isNextChrTrimable){
  749. this.state.quoting = false;
  750. this.state.wasQuoting = true;
  751. pos += quote.length - 1;
  752. continue;
  753. }else if(relax_quotes === false){
  754. const err = this.__error(
  755. new CsvError('CSV_INVALID_CLOSING_QUOTE', [
  756. 'Invalid Closing Quote:',
  757. `got "${String.fromCharCode(nextChr)}"`,
  758. `at line ${this.info.lines}`,
  759. 'instead of delimiter, record delimiter, trimable character',
  760. '(if activated) or comment',
  761. ], this.options, this.__infoField())
  762. );
  763. if(err !== undefined) return err;
  764. }else {
  765. this.state.quoting = false;
  766. this.state.wasQuoting = true;
  767. this.state.field.prepend(quote);
  768. pos += quote.length - 1;
  769. }
  770. }else {
  771. if(this.state.field.length !== 0){
  772. // In relax_quotes mode, treat opening quote preceded by chrs as regular
  773. if(relax_quotes === false){
  774. const info = this.__infoField();
  775. const bom = Object.keys(boms).map(b => boms[b].equals(this.state.field.toString()) ? b : false).filter(Boolean)[0];
  776. const err = this.__error(
  777. new CsvError('INVALID_OPENING_QUOTE', [
  778. 'Invalid Opening Quote:',
  779. `a quote is found on field ${JSON.stringify(info.column)} at line ${info.lines}, value is ${JSON.stringify(this.state.field.toString(encoding))}`,
  780. bom ? `(${bom} bom)` : undefined
  781. ], this.options, info, {
  782. field: this.state.field,
  783. })
  784. );
  785. if(err !== undefined) return err;
  786. }
  787. }else {
  788. this.state.quoting = true;
  789. pos += quote.length - 1;
  790. continue;
  791. }
  792. }
  793. }
  794. if(this.state.quoting === false){
  795. const recordDelimiterLength = this.__isRecordDelimiter(chr, buf, pos);
  796. if(recordDelimiterLength !== 0){
  797. // Do not emit comments which take a full line
  798. const skipCommentLine = this.state.commenting && (this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0);
  799. if(skipCommentLine){
  800. this.info.comment_lines++;
  801. // Skip full comment line
  802. }else {
  803. // Activate records emition if above from_line
  804. if(this.state.enabled === false && this.info.lines + (this.state.wasRowDelimiter === true ? 1: 0) >= from_line){
  805. this.state.enabled = true;
  806. this.__resetField();
  807. this.__resetRecord();
  808. pos += recordDelimiterLength - 1;
  809. continue;
  810. }
  811. // Skip if line is empty and skip_empty_lines activated
  812. if(skip_empty_lines === true && this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0){
  813. this.info.empty_lines++;
  814. pos += recordDelimiterLength - 1;
  815. continue;
  816. }
  817. this.info.bytes = this.state.bufBytesStart + pos;
  818. const errField = this.__onField();
  819. if(errField !== undefined) return errField;
  820. this.info.bytes = this.state.bufBytesStart + pos + recordDelimiterLength;
  821. const errRecord = this.__onRecord(push);
  822. if(errRecord !== undefined) return errRecord;
  823. if(to !== -1 && this.info.records >= to){
  824. this.state.stop = true;
  825. close();
  826. return;
  827. }
  828. }
  829. this.state.commenting = false;
  830. pos += recordDelimiterLength - 1;
  831. continue;
  832. }
  833. if(this.state.commenting){
  834. continue;
  835. }
  836. const commentCount = comment === null ? 0 : this.__compareBytes(comment, buf, pos, chr);
  837. if(commentCount !== 0 && (comment_no_infix === false || this.state.field.length === 0)){
  838. this.state.commenting = true;
  839. continue;
  840. }
  841. const delimiterLength = this.__isDelimiter(buf, pos, chr);
  842. if(delimiterLength !== 0){
  843. this.info.bytes = this.state.bufBytesStart + pos;
  844. const errField = this.__onField();
  845. if(errField !== undefined) return errField;
  846. pos += delimiterLength - 1;
  847. continue;
  848. }
  849. }
  850. }
  851. if(this.state.commenting === false){
  852. if(max_record_size !== 0 && this.state.record_length + this.state.field.length > max_record_size){
  853. return this.__error(
  854. new CsvError('CSV_MAX_RECORD_SIZE', [
  855. 'Max Record Size:',
  856. 'record exceed the maximum number of tolerated bytes',
  857. `of ${max_record_size}`,
  858. `at line ${this.info.lines}`,
  859. ], this.options, this.__infoField())
  860. );
  861. }
  862. }
  863. const lappend = ltrim === false || this.state.quoting === true || this.state.field.length !== 0 || !this.__isCharTrimable(buf, pos);
  864. // rtrim in non quoting is handle in __onField
  865. const rappend = rtrim === false || this.state.wasQuoting === false;
  866. if(lappend === true && rappend === true){
  867. this.state.field.append(chr);
  868. }else if(rtrim === true && !this.__isCharTrimable(buf, pos)){
  869. return this.__error(
  870. new CsvError('CSV_NON_TRIMABLE_CHAR_AFTER_CLOSING_QUOTE', [
  871. 'Invalid Closing Quote:',
  872. 'found non trimable byte after quote',
  873. `at line ${this.info.lines}`,
  874. ], this.options, this.__infoField())
  875. );
  876. }else {
  877. if(lappend === false){
  878. pos += this.__isCharTrimable(buf, pos) - 1;
  879. }
  880. continue;
  881. }
  882. }
  883. if(end === true){
  884. // Ensure we are not ending in a quoting state
  885. if(this.state.quoting === true){
  886. const err = this.__error(
  887. new CsvError('CSV_QUOTE_NOT_CLOSED', [
  888. 'Quote Not Closed:',
  889. `the parsing is finished with an opening quote at line ${this.info.lines}`,
  890. ], this.options, this.__infoField())
  891. );
  892. if(err !== undefined) return err;
  893. }else {
  894. // Skip last line if it has no characters
  895. if(this.state.wasQuoting === true || this.state.record.length !== 0 || this.state.field.length !== 0){
  896. this.info.bytes = this.state.bufBytesStart + pos;
  897. const errField = this.__onField();
  898. if(errField !== undefined) return errField;
  899. const errRecord = this.__onRecord(push);
  900. if(errRecord !== undefined) return errRecord;
  901. }else if(this.state.wasRowDelimiter === true){
  902. this.info.empty_lines++;
  903. }else if(this.state.commenting === true){
  904. this.info.comment_lines++;
  905. }
  906. }
  907. }else {
  908. this.state.bufBytesStart += pos;
  909. this.state.previousBuf = buf.slice(pos);
  910. }
  911. if(this.state.wasRowDelimiter === true){
  912. this.info.lines++;
  913. this.state.wasRowDelimiter = false;
  914. }
  915. },
  916. __onRecord: function(push){
  917. const {columns, group_columns_by_name, encoding, info, from, relax_column_count, relax_column_count_less, relax_column_count_more, raw, skip_records_with_empty_values} = this.options;
  918. const {enabled, record} = this.state;
  919. if(enabled === false){
  920. return this.__resetRecord();
  921. }
  922. // Convert the first line into column names
  923. const recordLength = record.length;
  924. if(columns === true){
  925. if(skip_records_with_empty_values === true && isRecordEmpty(record)){
  926. this.__resetRecord();
  927. return;
  928. }
  929. return this.__firstLineToColumns(record);
  930. }
  931. if(columns === false && this.info.records === 0){
  932. this.state.expectedRecordLength = recordLength;
  933. }
  934. if(recordLength !== this.state.expectedRecordLength){
  935. const err = columns === false ?
  936. new CsvError('CSV_RECORD_INCONSISTENT_FIELDS_LENGTH', [
  937. 'Invalid Record Length:',
  938. `expect ${this.state.expectedRecordLength},`,
  939. `got ${recordLength} on line ${this.info.lines}`,
  940. ], this.options, this.__infoField(), {
  941. record: record,
  942. })
  943. :
  944. new CsvError('CSV_RECORD_INCONSISTENT_COLUMNS', [
  945. 'Invalid Record Length:',
  946. `columns length is ${columns.length},`, // rename columns
  947. `got ${recordLength} on line ${this.info.lines}`,
  948. ], this.options, this.__infoField(), {
  949. record: record,
  950. });
  951. if(relax_column_count === true ||
  952. (relax_column_count_less === true && recordLength < this.state.expectedRecordLength) ||
  953. (relax_column_count_more === true && recordLength > this.state.expectedRecordLength)){
  954. this.info.invalid_field_length++;
  955. this.state.error = err;
  956. // Error is undefined with skip_records_with_error
  957. }else {
  958. const finalErr = this.__error(err);
  959. if(finalErr) return finalErr;
  960. }
  961. }
  962. if(skip_records_with_empty_values === true && isRecordEmpty(record)){
  963. this.__resetRecord();
  964. return;
  965. }
  966. if(this.state.recordHasError === true){
  967. this.__resetRecord();
  968. this.state.recordHasError = false;
  969. return;
  970. }
  971. this.info.records++;
  972. if(from === 1 || this.info.records >= from){
  973. const {objname} = this.options;
  974. // With columns, records are object
  975. if(columns !== false){
  976. const obj = {};
  977. // Transform record array to an object
  978. for(let i = 0, l = record.length; i < l; i++){
  979. if(columns[i] === undefined || columns[i].disabled) continue;
  980. // Turn duplicate columns into an array
  981. if (group_columns_by_name === true && obj[columns[i].name] !== undefined) {
  982. if (Array.isArray(obj[columns[i].name])) {
  983. obj[columns[i].name] = obj[columns[i].name].concat(record[i]);
  984. } else {
  985. obj[columns[i].name] = [obj[columns[i].name], record[i]];
  986. }
  987. } else {
  988. obj[columns[i].name] = record[i];
  989. }
  990. }
  991. // Without objname (default)
  992. if(raw === true || info === true){
  993. const extRecord = Object.assign(
  994. {record: obj},
  995. (raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}),
  996. (info === true ? {info: this.__infoRecord()}: {})
  997. );
  998. const err = this.__push(
  999. objname === undefined ? extRecord : [obj[objname], extRecord]
  1000. , push);
  1001. if(err){
  1002. return err;
  1003. }
  1004. }else {
  1005. const err = this.__push(
  1006. objname === undefined ? obj : [obj[objname], obj]
  1007. , push);
  1008. if(err){
  1009. return err;
  1010. }
  1011. }
  1012. // Without columns, records are array
  1013. }else {
  1014. if(raw === true || info === true){
  1015. const extRecord = Object.assign(
  1016. {record: record},
  1017. raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {},
  1018. info === true ? {info: this.__infoRecord()}: {}
  1019. );
  1020. const err = this.__push(
  1021. objname === undefined ? extRecord : [record[objname], extRecord]
  1022. , push);
  1023. if(err){
  1024. return err;
  1025. }
  1026. }else {
  1027. const err = this.__push(
  1028. objname === undefined ? record : [record[objname], record]
  1029. , push);
  1030. if(err){
  1031. return err;
  1032. }
  1033. }
  1034. }
  1035. }
  1036. this.__resetRecord();
  1037. },
  1038. __firstLineToColumns: function(record){
  1039. const {firstLineToHeaders} = this.state;
  1040. try{
  1041. const headers = firstLineToHeaders === undefined ? record : firstLineToHeaders.call(null, record);
  1042. if(!Array.isArray(headers)){
  1043. return this.__error(
  1044. new CsvError('CSV_INVALID_COLUMN_MAPPING', [
  1045. 'Invalid Column Mapping:',
  1046. 'expect an array from column function,',
  1047. `got ${JSON.stringify(headers)}`
  1048. ], this.options, this.__infoField(), {
  1049. headers: headers,
  1050. })
  1051. );
  1052. }
  1053. const normalizedHeaders = normalize_columns_array(headers);
  1054. this.state.expectedRecordLength = normalizedHeaders.length;
  1055. this.options.columns = normalizedHeaders;
  1056. this.__resetRecord();
  1057. return;
  1058. }catch(err){
  1059. return err;
  1060. }
  1061. },
  1062. __resetRecord: function(){
  1063. if(this.options.raw === true){
  1064. this.state.rawBuffer.reset();
  1065. }
  1066. this.state.error = undefined;
  1067. this.state.record = [];
  1068. this.state.record_length = 0;
  1069. },
  1070. __onField: function(){
  1071. const {cast, encoding, rtrim, max_record_size} = this.options;
  1072. const {enabled, wasQuoting} = this.state;
  1073. // Short circuit for the from_line options
  1074. if(enabled === false){
  1075. return this.__resetField();
  1076. }
  1077. let field = this.state.field.toString(encoding);
  1078. if(rtrim === true && wasQuoting === false){
  1079. field = field.trimRight();
  1080. }
  1081. if(cast === true){
  1082. const [err, f] = this.__cast(field);
  1083. if(err !== undefined) return err;
  1084. field = f;
  1085. }
  1086. this.state.record.push(field);
  1087. // Increment record length if record size must not exceed a limit
  1088. if(max_record_size !== 0 && typeof field === 'string'){
  1089. this.state.record_length += field.length;
  1090. }
  1091. this.__resetField();
  1092. },
  1093. __resetField: function(){
  1094. this.state.field.reset();
  1095. this.state.wasQuoting = false;
  1096. },
  1097. __push: function(record, push){
  1098. const {on_record} = this.options;
  1099. if(on_record !== undefined){
  1100. const info = this.__infoRecord();
  1101. try{
  1102. record = on_record.call(null, record, info);
  1103. }catch(err){
  1104. return err;
  1105. }
  1106. if(record === undefined || record === null){ return; }
  1107. }
  1108. push(record);
  1109. },
  1110. // Return a tuple with the error and the casted value
  1111. __cast: function(field){
  1112. const {columns, relax_column_count} = this.options;
  1113. const isColumns = Array.isArray(columns);
  1114. // Dont loose time calling cast
  1115. // because the final record is an object
  1116. // and this field can't be associated to a key present in columns
  1117. if(isColumns === true && relax_column_count && this.options.columns.length <= this.state.record.length){
  1118. return [undefined, undefined];
  1119. }
  1120. if(this.state.castField !== null){
  1121. try{
  1122. const info = this.__infoField();
  1123. return [undefined, this.state.castField.call(null, field, info)];
  1124. }catch(err){
  1125. return [err];
  1126. }
  1127. }
  1128. if(this.__isFloat(field)){
  1129. return [undefined, parseFloat(field)];
  1130. }else if(this.options.cast_date !== false){
  1131. const info = this.__infoField();
  1132. return [undefined, this.options.cast_date.call(null, field, info)];
  1133. }
  1134. return [undefined, field];
  1135. },
  1136. // Helper to test if a character is a space or a line delimiter
  1137. __isCharTrimable: function(buf, pos){
  1138. const isTrim = (buf, pos) => {
  1139. const {timchars} = this.state;
  1140. loop1: for(let i = 0; i < timchars.length; i++){
  1141. const timchar = timchars[i];
  1142. for(let j = 0; j < timchar.length; j++){
  1143. if(timchar[j] !== buf[pos+j]) continue loop1;
  1144. }
  1145. return timchar.length;
  1146. }
  1147. return 0;
  1148. };
  1149. return isTrim(buf, pos);
  1150. },
  1151. // Keep it in case we implement the `cast_int` option
  1152. // __isInt(value){
  1153. // // return Number.isInteger(parseInt(value))
  1154. // // return !isNaN( parseInt( obj ) );
  1155. // return /^(\-|\+)?[1-9][0-9]*$/.test(value)
  1156. // }
  1157. __isFloat: function(value){
  1158. return (value - parseFloat(value) + 1) >= 0; // Borrowed from jquery
  1159. },
  1160. __compareBytes: function(sourceBuf, targetBuf, targetPos, firstByte){
  1161. if(sourceBuf[0] !== firstByte) return 0;
  1162. const sourceLength = sourceBuf.length;
  1163. for(let i = 1; i < sourceLength; i++){
  1164. if(sourceBuf[i] !== targetBuf[targetPos+i]) return 0;
  1165. }
  1166. return sourceLength;
  1167. },
  1168. __isDelimiter: function(buf, pos, chr){
  1169. const {delimiter, ignore_last_delimiters} = this.options;
  1170. if(ignore_last_delimiters === true && this.state.record.length === this.options.columns.length - 1){
  1171. return 0;
  1172. }else if(ignore_last_delimiters !== false && typeof ignore_last_delimiters === 'number' && this.state.record.length === ignore_last_delimiters - 1){
  1173. return 0;
  1174. }
  1175. loop1: for(let i = 0; i < delimiter.length; i++){
  1176. const del = delimiter[i];
  1177. if(del[0] === chr){
  1178. for(let j = 1; j < del.length; j++){
  1179. if(del[j] !== buf[pos+j]) continue loop1;
  1180. }
  1181. return del.length;
  1182. }
  1183. }
  1184. return 0;
  1185. },
  1186. __isRecordDelimiter: function(chr, buf, pos){
  1187. const {record_delimiter} = this.options;
  1188. const recordDelimiterLength = record_delimiter.length;
  1189. loop1: for(let i = 0; i < recordDelimiterLength; i++){
  1190. const rd = record_delimiter[i];
  1191. const rdLength = rd.length;
  1192. if(rd[0] !== chr){
  1193. continue;
  1194. }
  1195. for(let j = 1; j < rdLength; j++){
  1196. if(rd[j] !== buf[pos+j]){
  1197. continue loop1;
  1198. }
  1199. }
  1200. return rd.length;
  1201. }
  1202. return 0;
  1203. },
  1204. __isEscape: function(buf, pos, chr){
  1205. const {escape} = this.options;
  1206. if(escape === null) return false;
  1207. const l = escape.length;
  1208. if(escape[0] === chr){
  1209. for(let i = 0; i < l; i++){
  1210. if(escape[i] !== buf[pos+i]){
  1211. return false;
  1212. }
  1213. }
  1214. return true;
  1215. }
  1216. return false;
  1217. },
  1218. __isQuote: function(buf, pos){
  1219. const {quote} = this.options;
  1220. if(quote === null) return false;
  1221. const l = quote.length;
  1222. for(let i = 0; i < l; i++){
  1223. if(quote[i] !== buf[pos+i]){
  1224. return false;
  1225. }
  1226. }
  1227. return true;
  1228. },
  1229. __autoDiscoverRecordDelimiter: function(buf, pos){
  1230. const { encoding } = this.options;
  1231. // Note, we don't need to cache this information in state,
  1232. // It is only called on the first line until we find out a suitable
  1233. // record delimiter.
  1234. const rds = [
  1235. // Important, the windows line ending must be before mac os 9
  1236. Buffer.from('\r\n', encoding),
  1237. Buffer.from('\n', encoding),
  1238. Buffer.from('\r', encoding),
  1239. ];
  1240. loop: for(let i = 0; i < rds.length; i++){
  1241. const l = rds[i].length;
  1242. for(let j = 0; j < l; j++){
  1243. if(rds[i][j] !== buf[pos + j]){
  1244. continue loop;
  1245. }
  1246. }
  1247. this.options.record_delimiter.push(rds[i]);
  1248. this.state.recordDelimiterMaxLength = rds[i].length;
  1249. return rds[i].length;
  1250. }
  1251. return 0;
  1252. },
  1253. __error: function(msg){
  1254. const {encoding, raw, skip_records_with_error} = this.options;
  1255. const err = typeof msg === 'string' ? new Error(msg) : msg;
  1256. if(skip_records_with_error){
  1257. this.state.recordHasError = true;
  1258. if(this.options.on_skip !== undefined){
  1259. this.options.on_skip(err, raw ? this.state.rawBuffer.toString(encoding) : undefined);
  1260. }
  1261. // this.emit('skip', err, raw ? this.state.rawBuffer.toString(encoding) : undefined);
  1262. return undefined;
  1263. }else {
  1264. return err;
  1265. }
  1266. },
  1267. __infoDataSet: function(){
  1268. return {
  1269. ...this.info,
  1270. columns: this.options.columns
  1271. };
  1272. },
  1273. __infoRecord: function(){
  1274. const {columns, raw, encoding} = this.options;
  1275. return {
  1276. ...this.__infoDataSet(),
  1277. error: this.state.error,
  1278. header: columns === true,
  1279. index: this.state.record.length,
  1280. raw: raw ? this.state.rawBuffer.toString(encoding) : undefined
  1281. };
  1282. },
  1283. __infoField: function(){
  1284. const {columns} = this.options;
  1285. const isColumns = Array.isArray(columns);
  1286. return {
  1287. ...this.__infoRecord(),
  1288. column: isColumns === true ?
  1289. (columns.length > this.state.record.length ?
  1290. columns[this.state.record.length].name :
  1291. null
  1292. ) :
  1293. this.state.record.length,
  1294. quoting: this.state.wasQuoting,
  1295. };
  1296. }
  1297. };
  1298. };
  1299. const parse = function(data, opts={}){
  1300. if(typeof data === 'string'){
  1301. data = Buffer.from(data);
  1302. }
  1303. const records = opts && opts.objname ? {} : [];
  1304. const parser = transform(opts);
  1305. const push = (record) => {
  1306. if(parser.options.objname === undefined)
  1307. records.push(record);
  1308. else {
  1309. records[record[0]] = record[1];
  1310. }
  1311. };
  1312. const close = () => {};
  1313. const err1 = parser.parse(data, false, push, close);
  1314. if(err1 !== undefined) throw err1;
  1315. const err2 = parser.parse(undefined, true, push, close);
  1316. if(err2 !== undefined) throw err2;
  1317. return records;
  1318. };
  1319. exports.CsvError = CsvError;
  1320. exports.parse = parse;