| | | 1 | | using ValidateLib.TabularData.Parsing; |
| | | 2 | | using ValidateLib.Metadata.Descriptors; |
| | | 3 | | |
| | | 4 | | namespace CSVReaderTests |
| | | 5 | | { |
| | | 6 | | public class RowParserTests |
| | | 7 | | { |
| | | 8 | | public readonly string testFilesDirectory; |
| | 1 | 9 | | private DialectDescriptor defaultDialect = new DialectDescriptor(); |
| | | 10 | | private Flags defaultFlags; |
| | 1 | 11 | | public RowParserTests() |
| | 1 | 12 | | { |
| | 1 | 13 | | testFilesDirectory = Path.Combine(GetProjectDirectory(), "TestFiles", "RowParserTestFiles"); |
| | 1 | 14 | | defaultFlags = FlagsCreator.ExtractFlagsFromDialectDescriptor(defaultDialect); |
| | 1 | 15 | | } |
| | | 16 | | static string GetProjectDirectory() |
| | 1 | 17 | | { |
| | 1 | 18 | | string? currentDirectory = Directory.GetCurrentDirectory(); |
| | | 19 | | |
| | 1 | 20 | | while (!string.IsNullOrEmpty(currentDirectory)) |
| | 1 | 21 | | { |
| | 1 | 22 | | string[] projectFiles = Directory.GetFiles(currentDirectory, "*.csproj"); |
| | | 23 | | |
| | 1 | 24 | | if (projectFiles.Length > 0) |
| | 1 | 25 | | return currentDirectory; |
| | 1 | 26 | | currentDirectory = Directory.GetParent(currentDirectory)?.FullName; |
| | 1 | 27 | | } |
| | | 28 | | |
| | 0 | 29 | | throw new Exception("Could not find project directory."); |
| | | 30 | | |
| | | 31 | | |
| | 1 | 32 | | } |
| | | 33 | | |
| | | 34 | | [Fact] |
| | | 35 | | public void SimpleTestWithDefaulDialect() |
| | 1 | 36 | | { |
| | 1 | 37 | | int inputCount = 3; |
| | 1 | 38 | | var inputs = new string[] |
| | 1 | 39 | | { |
| | 1 | 40 | | "countryCode,latitude,longitude,name", |
| | 1 | 41 | | "AD,42.546245,1.601554,Andorra", |
| | 1 | 42 | | "AF,33.93911,67.709953,Afghanistan" |
| | 1 | 43 | | |
| | 1 | 44 | | }; |
| | | 45 | | |
| | 1 | 46 | | List<string[]> expectedOutputs = new List<string[]>() |
| | 1 | 47 | | { |
| | 1 | 48 | | new string[] |
| | 1 | 49 | | { |
| | 1 | 50 | | "countryCode", |
| | 1 | 51 | | "latitude", |
| | 1 | 52 | | "longitude", |
| | 1 | 53 | | "name" |
| | 1 | 54 | | }, |
| | 1 | 55 | | new string[] |
| | 1 | 56 | | { |
| | 1 | 57 | | "AD", |
| | 1 | 58 | | "42.546245", |
| | 1 | 59 | | "1.601554", |
| | 1 | 60 | | "Andorra" |
| | 1 | 61 | | }, |
| | 1 | 62 | | new string[] |
| | 1 | 63 | | { |
| | 1 | 64 | | "AF", |
| | 1 | 65 | | "33.93911", |
| | 1 | 66 | | "67.709953", |
| | 1 | 67 | | "Afghanistan" |
| | 1 | 68 | | } |
| | 1 | 69 | | }; |
| | | 70 | | |
| | 1 | 71 | | RowParser rowParser = new RowParser(defaultFlags); |
| | | 72 | | |
| | 1 | 73 | | for (int inputNumber = 0; inputNumber < inputCount; inputNumber++) |
| | 1 | 74 | | { |
| | 1 | 75 | | var cellValues = rowParser.ParseRow(inputs[inputNumber]); |
| | 1 | 76 | | for (int i = 0 ; i < cellValues.Count; i++) |
| | 1 | 77 | | { |
| | 1 | 78 | | Assert.Equal(expectedOutputs[inputNumber][i], cellValues[i]); |
| | 1 | 79 | | } |
| | 1 | 80 | | } |
| | 1 | 81 | | } |
| | | 82 | | |
| | | 83 | | [Fact] |
| | | 84 | | public void SimpleTestWithDefaulDialectWithQuotedValue() |
| | 1 | 85 | | { |
| | 1 | 86 | | var expectedOutput = new string[] |
| | 1 | 87 | | { |
| | 1 | 88 | | |
| | 1 | 89 | | "AE", |
| | 1 | 90 | | "23.424076", |
| | 1 | 91 | | "53.847818", |
| | 1 | 92 | | "United Arab Emirates", |
| | 1 | 93 | | }; |
| | | 94 | | |
| | 1 | 95 | | string input = "AE,23.424076,53.847818,\"United Arab Emirates\""; |
| | 1 | 96 | | RowParser rowParser = new RowParser(defaultFlags); |
| | | 97 | | |
| | 1 | 98 | | List<string> cellValues = rowParser.ParseRow(input); |
| | | 99 | | |
| | 1 | 100 | | for(int i = 0 ; i < cellValues.Count; i++) |
| | 1 | 101 | | { |
| | 1 | 102 | | Assert.Equal(expectedOutput[i], cellValues[i]); |
| | 1 | 103 | | } |
| | | 104 | | |
| | 1 | 105 | | } |
| | | 106 | | |
| | | 107 | | [Fact] |
| | | 108 | | public void EscapedEscapedChar() |
| | 1 | 109 | | { |
| | 1 | 110 | | var expectedOutput = new string[] |
| | 1 | 111 | | { |
| | 1 | 112 | | |
| | 1 | 113 | | "\"Good joke\"", |
| | 1 | 114 | | "really!" |
| | 1 | 115 | | }; |
| | | 116 | | |
| | 1 | 117 | | string input = "\"\"Good joke\"\",really!"; |
| | 1 | 118 | | RowParser rowParser = new RowParser(defaultFlags); |
| | | 119 | | |
| | 1 | 120 | | List<string> cellValues = rowParser.ParseRow(input); |
| | | 121 | | |
| | 1 | 122 | | for (int i = 0; i < cellValues.Count; i++) |
| | 1 | 123 | | { |
| | 1 | 124 | | Assert.Equal(expectedOutput[i], cellValues[i]); |
| | 1 | 125 | | } |
| | | 126 | | |
| | 1 | 127 | | } |
| | | 128 | | |
| | | 129 | | [Fact] |
| | | 130 | | public void EscapedNewlineCharacter() |
| | 1 | 131 | | { |
| | 1 | 132 | | var expectedOutput = new string[] |
| | 1 | 133 | | { |
| | 1 | 134 | | |
| | 1 | 135 | | "Good joke\nNEWLINE", |
| | 1 | 136 | | "Really!" |
| | 1 | 137 | | }; |
| | | 138 | | |
| | 1 | 139 | | string input = "Good joke\\\nNEWLINE,Really!"; |
| | 1 | 140 | | var dialect = new DialectDescriptor(); |
| | 1 | 141 | | dialect!.doubleQuote!._value = false; |
| | 1 | 142 | | var flags = FlagsCreator.ExtractFlagsFromDialectDescriptor(dialect); |
| | 1 | 143 | | RowParser rowParser = new RowParser(flags); |
| | | 144 | | |
| | 1 | 145 | | List<string> cellValues = rowParser.ParseRow(input); |
| | | 146 | | |
| | 1 | 147 | | for (int i = 0; i < cellValues.Count; i++) |
| | 1 | 148 | | { |
| | 1 | 149 | | Assert.Equal(expectedOutput[i], cellValues[i]); |
| | 1 | 150 | | } |
| | | 151 | | |
| | 1 | 152 | | } |
| | | 153 | | |
| | | 154 | | |
| | | 155 | | [Fact] |
| | | 156 | | public void TrimStart() |
| | 1 | 157 | | { |
| | 1 | 158 | | var expectedOutput = new string[] |
| | 1 | 159 | | { |
| | 1 | 160 | | |
| | 1 | 161 | | "TrimStart ", |
| | 1 | 162 | | "TrimStart\t", |
| | 1 | 163 | | "TrimStart\n" |
| | 1 | 164 | | }; |
| | | 165 | | |
| | 1 | 166 | | string input = " TrimStart , \t TrimStart\t, \n \t TrimStart\n"; |
| | 1 | 167 | | var dialect = new DialectDescriptor(); |
| | 1 | 168 | | dialect.trim!._value = "start"; |
| | 1 | 169 | | var flags = FlagsCreator.ExtractFlagsFromDialectDescriptor(dialect); |
| | 1 | 170 | | RowParser rowParser = new RowParser(flags); |
| | | 171 | | |
| | 1 | 172 | | List<string> cellValues = rowParser.ParseRow(input); |
| | | 173 | | |
| | 1 | 174 | | for (int i = 0; i < cellValues.Count; i++) |
| | 1 | 175 | | { |
| | 1 | 176 | | Assert.Equal(expectedOutput[i], cellValues[i]); |
| | 1 | 177 | | } |
| | | 178 | | |
| | 1 | 179 | | } |
| | | 180 | | |
| | | 181 | | [Fact] |
| | | 182 | | public void TrimEnd() |
| | 1 | 183 | | { |
| | 1 | 184 | | var expectedOutput = new string[] |
| | 1 | 185 | | { |
| | 1 | 186 | | |
| | 1 | 187 | | " TrimEnd", |
| | 1 | 188 | | "\tTrimEnd", |
| | 1 | 189 | | "\nTrimEnd" |
| | 1 | 190 | | }; |
| | | 191 | | |
| | 1 | 192 | | string input = " TrimEnd ,\tTrimEnd \t ,\nTrimEnd \n \t "; |
| | 1 | 193 | | var dialect = new DialectDescriptor(); |
| | 1 | 194 | | dialect.trim!._value = "end"; |
| | 1 | 195 | | var flags = FlagsCreator.ExtractFlagsFromDialectDescriptor(dialect); |
| | 1 | 196 | | RowParser rowParser = new RowParser(flags); |
| | | 197 | | |
| | 1 | 198 | | List<string> cellValues = rowParser.ParseRow(input); |
| | | 199 | | |
| | 1 | 200 | | for (int i = 0; i < cellValues.Count; i++) |
| | 1 | 201 | | { |
| | 1 | 202 | | Assert.Equal(expectedOutput[i], cellValues[i]); |
| | 1 | 203 | | } |
| | | 204 | | |
| | 1 | 205 | | } |
| | | 206 | | |
| | | 207 | | [Fact] |
| | | 208 | | public void QuotedDelimiter() |
| | 1 | 209 | | { |
| | 1 | 210 | | var expectedOutput = new string[] |
| | 1 | 211 | | { |
| | 1 | 212 | | |
| | 1 | 213 | | "Hey", |
| | 1 | 214 | | "Quoted,Delimiter", |
| | 1 | 215 | | }; |
| | | 216 | | |
| | 1 | 217 | | string input = "Hey,\"Quoted,Delimiter\""; |
| | 1 | 218 | | RowParser rowParser = new RowParser(defaultFlags); |
| | | 219 | | |
| | 1 | 220 | | List<string> cellValues = rowParser.ParseRow(input); |
| | | 221 | | |
| | 1 | 222 | | for (int i = 0; i < cellValues.Count; i++) |
| | 1 | 223 | | { |
| | 1 | 224 | | Assert.Equal(expectedOutput[i], cellValues[i]); |
| | 1 | 225 | | } |
| | | 226 | | |
| | 1 | 227 | | } |
| | | 228 | | |
| | | 229 | | [Fact] |
| | | 230 | | public void QuotedQuoteChar() |
| | 1 | 231 | | { |
| | 1 | 232 | | var expectedOutput = new string[] |
| | 1 | 233 | | { |
| | 1 | 234 | | "\"random sentence.\"", |
| | 1 | 235 | | "Hey", |
| | 1 | 236 | | "this is \"quoted quote char\" continue", |
| | 1 | 237 | | |
| | 1 | 238 | | }; |
| | | 239 | | |
| | 1 | 240 | | string input = "\"\"\"random sentence.\"\"\",Hey,\"this is \"\"quoted quote char\"\" continue\""; |
| | 1 | 241 | | RowParser rowParser = new RowParser(defaultFlags); |
| | | 242 | | |
| | 1 | 243 | | List<string> cellValues = rowParser.ParseRow(input); |
| | | 244 | | |
| | 1 | 245 | | for (int i = 0; i < cellValues.Count; i++) |
| | 1 | 246 | | { |
| | 1 | 247 | | Assert.Equal(expectedOutput[i], cellValues[i]); |
| | 1 | 248 | | } |
| | | 249 | | |
| | 1 | 250 | | } |
| | | 251 | | |
| | | 252 | | /* |
| | | 253 | | [Fact] |
| | | 254 | | public void SimpleTestWithDefaulDialectWithEscapedChar() |
| | | 255 | | { |
| | | 256 | | var testFilePath = Path.Combine(testFilesDirectory, "escaped.csv"); |
| | | 257 | | var expectedLines = new string[] |
| | | 258 | | { |
| | | 259 | | "countryCode,latitude,longitude,name", |
| | | 260 | | "AD,42.546245,1.601554,Andorra", |
| | | 261 | | "AE,23.424076,53.847818,\"United \"\"Arab\"\" Emirates\"", |
| | | 262 | | "AF,33.93911,67.709953,Afghanistan" |
| | | 263 | | |
| | | 264 | | }; |
| | | 265 | | |
| | | 266 | | using (FileStream fs = new FileStream(testFilePath, FileMode.Open, FileAccess.Read)) |
| | | 267 | | { |
| | | 268 | | CustomStreamReader csr = new CustomStreamReader(fs, 50, 6); |
| | | 269 | | |
| | | 270 | | RowReader rowReader = new RowReader(defaultFlags); |
| | | 271 | | foreach (var line in expectedLines) |
| | | 272 | | { |
| | | 273 | | Assert.Equal(line, rowReader.ReadRow(csr)); |
| | | 274 | | } |
| | | 275 | | } |
| | | 276 | | } |
| | | 277 | | |
| | | 278 | | [Fact] |
| | | 279 | | public void TestQuotedNewLineDefaulDialect() |
| | | 280 | | { |
| | | 281 | | var testFilePath = Path.Combine(testFilesDirectory, "newline.csv"); |
| | | 282 | | var expectedLines = new string[] |
| | | 283 | | { |
| | | 284 | | "countryCode,latitude,longitude,name", |
| | | 285 | | "AD,42.546245,1.601554,Andorra", |
| | | 286 | | "AE,23.424076,53.847818,\"United\r\n Arab Emirates\"", |
| | | 287 | | "AF,33.93911,67.709953,Afghanistan" |
| | | 288 | | |
| | | 289 | | }; |
| | | 290 | | |
| | | 291 | | using (FileStream fs = new FileStream(testFilePath, FileMode.Open, FileAccess.Read)) |
| | | 292 | | { |
| | | 293 | | CustomStreamReader csr = new CustomStreamReader(fs, 50, 6); |
| | | 294 | | |
| | | 295 | | RowReader rowReader = new RowReader(defaultFlags); |
| | | 296 | | foreach (var line in expectedLines) |
| | | 297 | | { |
| | | 298 | | Assert.Equal(line, rowReader.ReadRow(csr)); |
| | | 299 | | } |
| | | 300 | | } |
| | | 301 | | } |
| | | 302 | | |
| | | 303 | | [Fact] |
| | | 304 | | public void SimpleTestWithDialectDifferentEndOfLine() |
| | | 305 | | { |
| | | 306 | | var testFilePathCsv = Path.Combine(testFilesDirectory, "Dialect", "test01", "countries.csv"); |
| | | 307 | | var testFilePathMetadata = Path.Combine(testFilesDirectory, "Dialect", "test01", "countries-metadata.json"); |
| | | 308 | | |
| | | 309 | | var warnings = new List<Warning>(); |
| | | 310 | | |
| | | 311 | | var tableDescriptor = MetadataParserValidator.ProcessTable( |
| | | 312 | | warnings, |
| | | 313 | | testFilePathMetadata |
| | | 314 | | ); |
| | | 315 | | |
| | | 316 | | var expectedLines = new string[] |
| | | 317 | | { |
| | | 318 | | "countryCode,latitude,longitude,name", |
| | | 319 | | "AD,42.546245,1.601554,Andorra", |
| | | 320 | | "AE,23.424076,53.847818,\"United Arab Emirates\"", |
| | | 321 | | "AF,33.93911,67.709953,Afghanistan" |
| | | 322 | | |
| | | 323 | | }; |
| | | 324 | | |
| | | 325 | | using (FileStream fs = new FileStream(testFilePathCsv, FileMode.Open, FileAccess.Read)) |
| | | 326 | | { |
| | | 327 | | |
| | | 328 | | |
| | | 329 | | Flags flags = FlagsCreator.ExtractFlagsFromDialectDescriptor(tableDescriptor.dialect!._value!); |
| | | 330 | | RowReader rowReader = new RowReader(flags); |
| | | 331 | | |
| | | 332 | | CustomStreamReader csr = new CustomStreamReader(fs, 50, rowReader.MaximalLineTerminatorLength); |
| | | 333 | | foreach (var line in expectedLines) |
| | | 334 | | { |
| | | 335 | | Assert.Equal(line, rowReader.ReadRow(csr)); |
| | | 336 | | } |
| | | 337 | | } |
| | | 338 | | |
| | | 339 | | |
| | | 340 | | } |
| | | 341 | | |
| | | 342 | | |
| | | 343 | | [Fact] |
| | | 344 | | public void SimpleTestWithDialectDifferentQuoteChar() |
| | | 345 | | { |
| | | 346 | | var testFilePathCsv = Path.Combine(testFilesDirectory, "Dialect", "test02", "countries.csv"); |
| | | 347 | | var testFilePathMetadata = Path.Combine(testFilesDirectory, "Dialect", "test02", "countries-metadata.json"); |
| | | 348 | | |
| | | 349 | | var warnings = new List<Warning>(); |
| | | 350 | | |
| | | 351 | | var tableDescriptor = MetadataParserValidator.ProcessTable( |
| | | 352 | | warnings, |
| | | 353 | | testFilePathMetadata |
| | | 354 | | ); |
| | | 355 | | |
| | | 356 | | var expectedLines = new string[] |
| | | 357 | | { |
| | | 358 | | "countryCode,latitude,longitude,name", |
| | | 359 | | "AD,42.546245,1.601554,Andorra", |
| | | 360 | | "AE,23.424076,53.847818,QCHUnited\r\n Arab EmiratesQCH", |
| | | 361 | | "AF,33.93911,67.709953,Afghanistan" |
| | | 362 | | |
| | | 363 | | }; |
| | | 364 | | |
| | | 365 | | using (FileStream fs = new FileStream(testFilePathCsv, FileMode.Open, FileAccess.Read)) |
| | | 366 | | { |
| | | 367 | | |
| | | 368 | | |
| | | 369 | | Flags flags = FlagsCreator.ExtractFlagsFromDialectDescriptor(tableDescriptor.dialect!._value!); |
| | | 370 | | RowReader rowReader = new RowReader(flags); |
| | | 371 | | |
| | | 372 | | CustomStreamReader csr = new CustomStreamReader(fs, 50, rowReader.MaximalLineTerminatorLength); |
| | | 373 | | foreach (var line in expectedLines) |
| | | 374 | | { |
| | | 375 | | Assert.Equal(line, rowReader.ReadRow(csr)); |
| | | 376 | | } |
| | | 377 | | } |
| | | 378 | | |
| | | 379 | | |
| | | 380 | | } |
| | | 381 | | */ |
| | | 382 | | } |
| | | 383 | | } |