| | 1 | | using ValidateLib.TabularData.Parsing; |
| | 2 | | using ValidateLib.Metadata.Descriptors; |
| | 3 | |
|
| | 4 | | namespace CSVReaderTests |
| | 5 | | { |
| | 6 | | public class RowParserTests |
| | 7 | | { |
| | 8 | | public readonly string testFilesDirectory; |
| 1 | 9 | | private DialectDescriptor defaultDialect = new DialectDescriptor(); |
| | 10 | | private Flags defaultFlags; |
| 1 | 11 | | public RowParserTests() |
| 1 | 12 | | { |
| 1 | 13 | | testFilesDirectory = Path.Combine(GetProjectDirectory(), "TestFiles", "RowParserTestFiles"); |
| 1 | 14 | | defaultFlags = FlagsCreator.ExtractFlagsFromDialectDescriptor(defaultDialect); |
| 1 | 15 | | } |
| | 16 | | static string GetProjectDirectory() |
| 1 | 17 | | { |
| 1 | 18 | | string? currentDirectory = Directory.GetCurrentDirectory(); |
| | 19 | |
|
| 1 | 20 | | while (!string.IsNullOrEmpty(currentDirectory)) |
| 1 | 21 | | { |
| 1 | 22 | | string[] projectFiles = Directory.GetFiles(currentDirectory, "*.csproj"); |
| | 23 | |
|
| 1 | 24 | | if (projectFiles.Length > 0) |
| 1 | 25 | | return currentDirectory; |
| 1 | 26 | | currentDirectory = Directory.GetParent(currentDirectory)?.FullName; |
| 1 | 27 | | } |
| | 28 | |
|
| 0 | 29 | | throw new Exception("Could not find project directory."); |
| | 30 | |
|
| | 31 | |
|
| 1 | 32 | | } |
| | 33 | |
|
| | 34 | | [Fact] |
| | 35 | | public void SimpleTestWithDefaulDialect() |
| 1 | 36 | | { |
| 1 | 37 | | int inputCount = 3; |
| 1 | 38 | | var inputs = new string[] |
| 1 | 39 | | { |
| 1 | 40 | | "countryCode,latitude,longitude,name", |
| 1 | 41 | | "AD,42.546245,1.601554,Andorra", |
| 1 | 42 | | "AF,33.93911,67.709953,Afghanistan" |
| 1 | 43 | |
|
| 1 | 44 | | }; |
| | 45 | |
|
| 1 | 46 | | List<string[]> expectedOutputs = new List<string[]>() |
| 1 | 47 | | { |
| 1 | 48 | | new string[] |
| 1 | 49 | | { |
| 1 | 50 | | "countryCode", |
| 1 | 51 | | "latitude", |
| 1 | 52 | | "longitude", |
| 1 | 53 | | "name" |
| 1 | 54 | | }, |
| 1 | 55 | | new string[] |
| 1 | 56 | | { |
| 1 | 57 | | "AD", |
| 1 | 58 | | "42.546245", |
| 1 | 59 | | "1.601554", |
| 1 | 60 | | "Andorra" |
| 1 | 61 | | }, |
| 1 | 62 | | new string[] |
| 1 | 63 | | { |
| 1 | 64 | | "AF", |
| 1 | 65 | | "33.93911", |
| 1 | 66 | | "67.709953", |
| 1 | 67 | | "Afghanistan" |
| 1 | 68 | | } |
| 1 | 69 | | }; |
| | 70 | |
|
| 1 | 71 | | RowParser rowParser = new RowParser(defaultFlags); |
| | 72 | |
|
| 1 | 73 | | for (int inputNumber = 0; inputNumber < inputCount; inputNumber++) |
| 1 | 74 | | { |
| 1 | 75 | | var cellValues = rowParser.ParseRow(inputs[inputNumber]); |
| 1 | 76 | | for (int i = 0 ; i < cellValues.Count; i++) |
| 1 | 77 | | { |
| 1 | 78 | | Assert.Equal(expectedOutputs[inputNumber][i], cellValues[i]); |
| 1 | 79 | | } |
| 1 | 80 | | } |
| 1 | 81 | | } |
| | 82 | |
|
| | 83 | | [Fact] |
| | 84 | | public void SimpleTestWithDefaulDialectWithQuotedValue() |
| 1 | 85 | | { |
| 1 | 86 | | var expectedOutput = new string[] |
| 1 | 87 | | { |
| 1 | 88 | |
|
| 1 | 89 | | "AE", |
| 1 | 90 | | "23.424076", |
| 1 | 91 | | "53.847818", |
| 1 | 92 | | "United Arab Emirates", |
| 1 | 93 | | }; |
| | 94 | |
|
| 1 | 95 | | string input = "AE,23.424076,53.847818,\"United Arab Emirates\""; |
| 1 | 96 | | RowParser rowParser = new RowParser(defaultFlags); |
| | 97 | |
|
| 1 | 98 | | List<string> cellValues = rowParser.ParseRow(input); |
| | 99 | |
|
| 1 | 100 | | for(int i = 0 ; i < cellValues.Count; i++) |
| 1 | 101 | | { |
| 1 | 102 | | Assert.Equal(expectedOutput[i], cellValues[i]); |
| 1 | 103 | | } |
| | 104 | |
|
| 1 | 105 | | } |
| | 106 | |
|
| | 107 | | [Fact] |
| | 108 | | public void EscapedEscapedChar() |
| 1 | 109 | | { |
| 1 | 110 | | var expectedOutput = new string[] |
| 1 | 111 | | { |
| 1 | 112 | |
|
| 1 | 113 | | "\"Good joke\"", |
| 1 | 114 | | "really!" |
| 1 | 115 | | }; |
| | 116 | |
|
| 1 | 117 | | string input = "\"\"Good joke\"\",really!"; |
| 1 | 118 | | RowParser rowParser = new RowParser(defaultFlags); |
| | 119 | |
|
| 1 | 120 | | List<string> cellValues = rowParser.ParseRow(input); |
| | 121 | |
|
| 1 | 122 | | for (int i = 0; i < cellValues.Count; i++) |
| 1 | 123 | | { |
| 1 | 124 | | Assert.Equal(expectedOutput[i], cellValues[i]); |
| 1 | 125 | | } |
| | 126 | |
|
| 1 | 127 | | } |
| | 128 | |
|
| | 129 | | [Fact] |
| | 130 | | public void EscapedNewlineCharacter() |
| 1 | 131 | | { |
| 1 | 132 | | var expectedOutput = new string[] |
| 1 | 133 | | { |
| 1 | 134 | |
|
| 1 | 135 | | "Good joke\nNEWLINE", |
| 1 | 136 | | "Really!" |
| 1 | 137 | | }; |
| | 138 | |
|
| 1 | 139 | | string input = "Good joke\\\nNEWLINE,Really!"; |
| 1 | 140 | | var dialect = new DialectDescriptor(); |
| 1 | 141 | | dialect!.doubleQuote!._value = false; |
| 1 | 142 | | var flags = FlagsCreator.ExtractFlagsFromDialectDescriptor(dialect); |
| 1 | 143 | | RowParser rowParser = new RowParser(flags); |
| | 144 | |
|
| 1 | 145 | | List<string> cellValues = rowParser.ParseRow(input); |
| | 146 | |
|
| 1 | 147 | | for (int i = 0; i < cellValues.Count; i++) |
| 1 | 148 | | { |
| 1 | 149 | | Assert.Equal(expectedOutput[i], cellValues[i]); |
| 1 | 150 | | } |
| | 151 | |
|
| 1 | 152 | | } |
| | 153 | |
|
| | 154 | |
|
| | 155 | | [Fact] |
| | 156 | | public void TrimStart() |
| 1 | 157 | | { |
| 1 | 158 | | var expectedOutput = new string[] |
| 1 | 159 | | { |
| 1 | 160 | |
|
| 1 | 161 | | "TrimStart ", |
| 1 | 162 | | "TrimStart\t", |
| 1 | 163 | | "TrimStart\n" |
| 1 | 164 | | }; |
| | 165 | |
|
| 1 | 166 | | string input = " TrimStart , \t TrimStart\t, \n \t TrimStart\n"; |
| 1 | 167 | | var dialect = new DialectDescriptor(); |
| 1 | 168 | | dialect.trim!._value = "start"; |
| 1 | 169 | | var flags = FlagsCreator.ExtractFlagsFromDialectDescriptor(dialect); |
| 1 | 170 | | RowParser rowParser = new RowParser(flags); |
| | 171 | |
|
| 1 | 172 | | List<string> cellValues = rowParser.ParseRow(input); |
| | 173 | |
|
| 1 | 174 | | for (int i = 0; i < cellValues.Count; i++) |
| 1 | 175 | | { |
| 1 | 176 | | Assert.Equal(expectedOutput[i], cellValues[i]); |
| 1 | 177 | | } |
| | 178 | |
|
| 1 | 179 | | } |
| | 180 | |
|
| | 181 | | [Fact] |
| | 182 | | public void TrimEnd() |
| 1 | 183 | | { |
| 1 | 184 | | var expectedOutput = new string[] |
| 1 | 185 | | { |
| 1 | 186 | |
|
| 1 | 187 | | " TrimEnd", |
| 1 | 188 | | "\tTrimEnd", |
| 1 | 189 | | "\nTrimEnd" |
| 1 | 190 | | }; |
| | 191 | |
|
| 1 | 192 | | string input = " TrimEnd ,\tTrimEnd \t ,\nTrimEnd \n \t "; |
| 1 | 193 | | var dialect = new DialectDescriptor(); |
| 1 | 194 | | dialect.trim!._value = "end"; |
| 1 | 195 | | var flags = FlagsCreator.ExtractFlagsFromDialectDescriptor(dialect); |
| 1 | 196 | | RowParser rowParser = new RowParser(flags); |
| | 197 | |
|
| 1 | 198 | | List<string> cellValues = rowParser.ParseRow(input); |
| | 199 | |
|
| 1 | 200 | | for (int i = 0; i < cellValues.Count; i++) |
| 1 | 201 | | { |
| 1 | 202 | | Assert.Equal(expectedOutput[i], cellValues[i]); |
| 1 | 203 | | } |
| | 204 | |
|
| 1 | 205 | | } |
| | 206 | |
|
| | 207 | | [Fact] |
| | 208 | | public void QuotedDelimiter() |
| 1 | 209 | | { |
| 1 | 210 | | var expectedOutput = new string[] |
| 1 | 211 | | { |
| 1 | 212 | |
|
| 1 | 213 | | "Hey", |
| 1 | 214 | | "Quoted,Delimiter", |
| 1 | 215 | | }; |
| | 216 | |
|
| 1 | 217 | | string input = "Hey,\"Quoted,Delimiter\""; |
| 1 | 218 | | RowParser rowParser = new RowParser(defaultFlags); |
| | 219 | |
|
| 1 | 220 | | List<string> cellValues = rowParser.ParseRow(input); |
| | 221 | |
|
| 1 | 222 | | for (int i = 0; i < cellValues.Count; i++) |
| 1 | 223 | | { |
| 1 | 224 | | Assert.Equal(expectedOutput[i], cellValues[i]); |
| 1 | 225 | | } |
| | 226 | |
|
| 1 | 227 | | } |
| | 228 | |
|
| | 229 | | [Fact] |
| | 230 | | public void QuotedQuoteChar() |
| 1 | 231 | | { |
| 1 | 232 | | var expectedOutput = new string[] |
| 1 | 233 | | { |
| 1 | 234 | | "\"random sentence.\"", |
| 1 | 235 | | "Hey", |
| 1 | 236 | | "this is \"quoted quote char\" continue", |
| 1 | 237 | |
|
| 1 | 238 | | }; |
| | 239 | |
|
| 1 | 240 | | string input = "\"\"\"random sentence.\"\"\",Hey,\"this is \"\"quoted quote char\"\" continue\""; |
| 1 | 241 | | RowParser rowParser = new RowParser(defaultFlags); |
| | 242 | |
|
| 1 | 243 | | List<string> cellValues = rowParser.ParseRow(input); |
| | 244 | |
|
| 1 | 245 | | for (int i = 0; i < cellValues.Count; i++) |
| 1 | 246 | | { |
| 1 | 247 | | Assert.Equal(expectedOutput[i], cellValues[i]); |
| 1 | 248 | | } |
| | 249 | |
|
| 1 | 250 | | } |
| | 251 | |
|
| | 252 | | /* |
| | 253 | | [Fact] |
| | 254 | | public void SimpleTestWithDefaulDialectWithEscapedChar() |
| | 255 | | { |
| | 256 | | var testFilePath = Path.Combine(testFilesDirectory, "escaped.csv"); |
| | 257 | | var expectedLines = new string[] |
| | 258 | | { |
| | 259 | | "countryCode,latitude,longitude,name", |
| | 260 | | "AD,42.546245,1.601554,Andorra", |
| | 261 | | "AE,23.424076,53.847818,\"United \"\"Arab\"\" Emirates\"", |
| | 262 | | "AF,33.93911,67.709953,Afghanistan" |
| | 263 | |
|
| | 264 | | }; |
| | 265 | |
|
| | 266 | | using (FileStream fs = new FileStream(testFilePath, FileMode.Open, FileAccess.Read)) |
| | 267 | | { |
| | 268 | | CustomStreamReader csr = new CustomStreamReader(fs, 50, 6); |
| | 269 | |
|
| | 270 | | RowReader rowReader = new RowReader(defaultFlags); |
| | 271 | | foreach (var line in expectedLines) |
| | 272 | | { |
| | 273 | | Assert.Equal(line, rowReader.ReadRow(csr)); |
| | 274 | | } |
| | 275 | | } |
| | 276 | | } |
| | 277 | |
|
| | 278 | | [Fact] |
| | 279 | | public void TestQuotedNewLineDefaulDialect() |
| | 280 | | { |
| | 281 | | var testFilePath = Path.Combine(testFilesDirectory, "newline.csv"); |
| | 282 | | var expectedLines = new string[] |
| | 283 | | { |
| | 284 | | "countryCode,latitude,longitude,name", |
| | 285 | | "AD,42.546245,1.601554,Andorra", |
| | 286 | | "AE,23.424076,53.847818,\"United\r\n Arab Emirates\"", |
| | 287 | | "AF,33.93911,67.709953,Afghanistan" |
| | 288 | |
|
| | 289 | | }; |
| | 290 | |
|
| | 291 | | using (FileStream fs = new FileStream(testFilePath, FileMode.Open, FileAccess.Read)) |
| | 292 | | { |
| | 293 | | CustomStreamReader csr = new CustomStreamReader(fs, 50, 6); |
| | 294 | |
|
| | 295 | | RowReader rowReader = new RowReader(defaultFlags); |
| | 296 | | foreach (var line in expectedLines) |
| | 297 | | { |
| | 298 | | Assert.Equal(line, rowReader.ReadRow(csr)); |
| | 299 | | } |
| | 300 | | } |
| | 301 | | } |
| | 302 | |
|
| | 303 | | [Fact] |
| | 304 | | public void SimpleTestWithDialectDifferentEndOfLine() |
| | 305 | | { |
| | 306 | | var testFilePathCsv = Path.Combine(testFilesDirectory, "Dialect", "test01", "countries.csv"); |
| | 307 | | var testFilePathMetadata = Path.Combine(testFilesDirectory, "Dialect", "test01", "countries-metadata.json"); |
| | 308 | |
|
| | 309 | | var warnings = new List<Warning>(); |
| | 310 | |
|
| | 311 | | var tableDescriptor = MetadataParserValidator.ProcessTable( |
| | 312 | | warnings, |
| | 313 | | testFilePathMetadata |
| | 314 | | ); |
| | 315 | |
|
| | 316 | | var expectedLines = new string[] |
| | 317 | | { |
| | 318 | | "countryCode,latitude,longitude,name", |
| | 319 | | "AD,42.546245,1.601554,Andorra", |
| | 320 | | "AE,23.424076,53.847818,\"United Arab Emirates\"", |
| | 321 | | "AF,33.93911,67.709953,Afghanistan" |
| | 322 | |
|
| | 323 | | }; |
| | 324 | |
|
| | 325 | | using (FileStream fs = new FileStream(testFilePathCsv, FileMode.Open, FileAccess.Read)) |
| | 326 | | { |
| | 327 | |
|
| | 328 | |
|
| | 329 | | Flags flags = FlagsCreator.ExtractFlagsFromDialectDescriptor(tableDescriptor.dialect!._value!); |
| | 330 | | RowReader rowReader = new RowReader(flags); |
| | 331 | |
|
| | 332 | | CustomStreamReader csr = new CustomStreamReader(fs, 50, rowReader.MaximalLineTerminatorLength); |
| | 333 | | foreach (var line in expectedLines) |
| | 334 | | { |
| | 335 | | Assert.Equal(line, rowReader.ReadRow(csr)); |
| | 336 | | } |
| | 337 | | } |
| | 338 | |
|
| | 339 | |
|
| | 340 | | } |
| | 341 | |
|
| | 342 | |
|
| | 343 | | [Fact] |
| | 344 | | public void SimpleTestWithDialectDifferentQuoteChar() |
| | 345 | | { |
| | 346 | | var testFilePathCsv = Path.Combine(testFilesDirectory, "Dialect", "test02", "countries.csv"); |
| | 347 | | var testFilePathMetadata = Path.Combine(testFilesDirectory, "Dialect", "test02", "countries-metadata.json"); |
| | 348 | |
|
| | 349 | | var warnings = new List<Warning>(); |
| | 350 | |
|
| | 351 | | var tableDescriptor = MetadataParserValidator.ProcessTable( |
| | 352 | | warnings, |
| | 353 | | testFilePathMetadata |
| | 354 | | ); |
| | 355 | |
|
| | 356 | | var expectedLines = new string[] |
| | 357 | | { |
| | 358 | | "countryCode,latitude,longitude,name", |
| | 359 | | "AD,42.546245,1.601554,Andorra", |
| | 360 | | "AE,23.424076,53.847818,QCHUnited\r\n Arab EmiratesQCH", |
| | 361 | | "AF,33.93911,67.709953,Afghanistan" |
| | 362 | |
|
| | 363 | | }; |
| | 364 | |
|
| | 365 | | using (FileStream fs = new FileStream(testFilePathCsv, FileMode.Open, FileAccess.Read)) |
| | 366 | | { |
| | 367 | |
|
| | 368 | |
|
| | 369 | | Flags flags = FlagsCreator.ExtractFlagsFromDialectDescriptor(tableDescriptor.dialect!._value!); |
| | 370 | | RowReader rowReader = new RowReader(flags); |
| | 371 | |
|
| | 372 | | CustomStreamReader csr = new CustomStreamReader(fs, 50, rowReader.MaximalLineTerminatorLength); |
| | 373 | | foreach (var line in expectedLines) |
| | 374 | | { |
| | 375 | | Assert.Equal(line, rowReader.ReadRow(csr)); |
| | 376 | | } |
| | 377 | | } |
| | 378 | |
|
| | 379 | |
|
| | 380 | | } |
| | 381 | | */ |
| | 382 | | } |
| | 383 | | } |