Improve CSV importer

Summary:
The importer now supports all of the flags that the modern Neo4j CSV importer
supports.

Reviewers: teon.banek, ipaljak

Reviewed By: teon.banek

Subscribers: pullbot

Differential Revision: https://phabricator.memgraph.io/D2709

src/mg_import_csv.cpp

@@ -28,18 +28,11 @@ bool ValidateControlCharacter(const char *flagname, const std::string &value) {
   return true;
 }
 
-bool ValidateNoWhitespace(const char *flagname, const std::string &value) {
-  auto trimmed = utils::Trim(value);
-  if (trimmed.empty() && !value.empty()) {
-    printf("The argument '%s' cannot be only whitespace\n", flagname);
+bool ValidateIdTypeOptions(const char *flagname, const std::string &value) {
+  std::string upper = utils::ToUpperCase(utils::Trim(value));
+  if (upper != "STRING" && upper != "INTEGER") {
+    printf("Valid options for '%s' are: STRING/INTEGER\n", flagname);
     return false;
-  } else if (!trimmed.empty()) {
-    for (auto c : trimmed) {
-      if (std::isspace(c)) {
-        printf("The argument '%s' cannot contain whitespace\n", flagname);
-        return false;
-      }
-    }
   }
   return true;
 }
@@ -65,18 +58,39 @@ DEFINE_string(quote, "\"",
 DEFINE_validator(quote, &ValidateControlCharacter);
 DEFINE_bool(skip_duplicate_nodes, false,
             "Set to true to skip duplicate nodes instead of raising an error.");
+DEFINE_bool(skip_bad_relationships, false,
+            "Set to true to skip relationships that connect nodes that don't "
+            "exist instead of raising an error.");
+DEFINE_bool(ignore_empty_strings, false,
+            "Set to true to treat empty strings as null values.");
+DEFINE_bool(
+    ignore_extra_columns, false,
+    "Set to true to ignore columns that aren't specified in the header.");
+DEFINE_bool(trim_strings, false,
+            "Set to true to trim leading/trailing whitespace from all fields "
+            "that are loaded from the CSV file.");
+DEFINE_string(id_type, "STRING",
+              "Which data type should be used to store the supplied node IDs. "
+              "Possible options are: STRING/INTEGER");
+DEFINE_validator(id_type, &ValidateIdTypeOptions);
 // Arguments `--nodes` and `--relationships` can be input multiple times and are
 // handled with custom parsing.
-DEFINE_string(nodes, "", "CSV file containing graph nodes (vertices).");
-DEFINE_string(node_label, "",
-              "Specify additional label for nodes. To add multiple labels, "
-              "repeat the flag multiple times.");
-DEFINE_validator(node_label, &ValidateNoWhitespace);
-DEFINE_string(relationships, "",
-              "CSV file containing graph relationships (edges).");
-DEFINE_string(relationship_type, "",
-              "Overwrite the relationship type from csv with the given value.");
-DEFINE_validator(relationship_type, &ValidateNoWhitespace);
+DEFINE_string(
+    nodes, "",
+    "Files that should be parsed for nodes. The CSV header will be loaded from "
+    "the first supplied file, all other files supplied in a single flag will "
+    "be treated as data files. Additional labels can be specified for the node "
+    "files. The flag can be specified multiple times (useful for differently "
+    "formatted node files). The format of this argument is: "
+    "[<label>[:<label>]...=]<file>[,<file>][,<file>]...");
+DEFINE_string(
+    relationships, "",
+    "Files that should be parsed for relationships. The CSV header will be "
+    "loaded from the first supplied file, all other files supplied in a single "
+    "flag will be treated as data files. The relationship type can be "
+    "specified for the relationship files. The flag can be specified multiple "
+    "times (useful for differently formatted relationship files). The format "
+    "of this argument is: [<type>=]<file>[,<file>][,<file>]...");
 
 std::vector<std::string> ParseRepeatedFlag(const std::string &flagname,
                                            int argc, char *argv[]) {
@@ -123,7 +137,11 @@ bool operator==(const NodeId &a, const NodeId &b) {
 }
 
 std::ostream &operator<<(std::ostream &stream, const NodeId &node_id) {
-  return stream << node_id.id << "(" << node_id.id_space << ")";
+  if (!node_id.id_space.empty()) {
+    return stream << node_id.id << "(" << node_id.id_space << ")";
+  } else {
+    return stream << node_id.id;
+  }
 }
 
 namespace std {
@@ -307,6 +325,13 @@ std::pair<std::vector<std::string>, uint64_t> ReadRow(std::istream &stream) {
     }
   }
 
+  if (FLAGS_trim_strings) {
+    for (size_t i = 0; i < row.size(); ++i) {
+      std::string trimmed(utils::Trim(row[i]));
+      row[i] = std::move(trimmed);
+    }
+  }
+
   return {std::move(row), lines_count};
 }
 
@@ -331,20 +356,36 @@ std::pair<std::vector<Field>, uint64_t> ReadHeader(std::istream &stream) {
   return {std::move(fields), lines_count};
 }
 
+/// @throw LoadException
+int64_t StringToInt(const std::string &value) {
+  try {
+    return utils::ParseInt(value);
+  } catch (...) {
+    throw LoadException("'{}' isn't a valid integer", value);
+  }
+}
+
+/// @throw LoadException
+double StringToDouble(const std::string &value) {
+  try {
+    return utils::ParseDouble(value);
+  } catch (...) {
+    throw LoadException("'{}' isn't a valid floating-point value", value);
+  }
+}
+
 /// @throw LoadException
 storage::PropertyValue StringToValue(const std::string &str,
                                      const std::string &type) {
-  // Empty string signifies Null.
-  if (str.empty()) return storage::PropertyValue();
+  if (FLAGS_ignore_empty_strings && str.empty())
+    return storage::PropertyValue();
   auto convert = [](const auto &str, const auto &type) {
-    if (type == "int" || type == "long" || type == "byte" || type == "short") {
-      std::istringstream ss(str);
-      int64_t val;
-      ss >> val;
-      return storage::PropertyValue(val);
+    if (type == "integer" || type == "int" || type == "long" ||
+        type == "byte" || type == "short") {
+      return storage::PropertyValue(StringToInt(str));
     } else if (type == "float" || type == "double") {
-      return storage::PropertyValue(utils::ParseDouble(str));
-    } else if (type == "boolean") {
+      return storage::PropertyValue(StringToDouble(str));
+    } else if (type == "boolean" || type == "bool") {
       if (utils::ToLowerCase(str) == "true") {
         return storage::PropertyValue(true);
       } else {
@@ -363,7 +404,7 @@ storage::PropertyValue StringToValue(const std::string &str,
   std::vector<storage::PropertyValue> array;
   array.reserve(elems.size());
   for (const auto &elem : elems) {
-    array.push_back(convert(std::string(utils::Trim(elem)), elem_type));
+    array.push_back(convert(elem, elem_type));
   }
   return storage::PropertyValue(std::move(array));
 }
@@ -394,36 +435,47 @@ void ProcessNodeRow(storage::Storage *store, const std::vector<Field> &fields,
   auto node = acc.CreateVertex();
   for (size_t i = 0; i < row.size(); ++i) {
     const auto &field = fields[i];
-    std::string value(utils::Trim(row[i]));
+    const auto &value = row[i];
     if (utils::StartsWith(field.type, "ID")) {
       if (id) throw LoadException("Only one node ID must be specified");
+      if (FLAGS_id_type == "INTEGER") {
+        // Call `StringToInt` to verify that the ID is a valid integer.
+        StringToInt(value);
+      }
       NodeId node_id{value, GetIdSpace(field.type)};
       auto it = node_id_map->find(node_id);
       if (it != node_id_map->end()) {
         if (FLAGS_skip_duplicate_nodes) {
-          LOG(WARNING) << "Skipping duplicate node with id '" << node_id << "'";
+          LOG(WARNING) << "Skipping duplicate node with ID '" << node_id << "'";
           return;
         } else {
-          throw LoadException("Node with id '{}' already exists", node_id);
+          throw LoadException("Node with ID '{}' already exists", node_id);
         }
       }
       node_id_map->emplace(node_id, node.Gid());
-      auto node_property = node.SetProperty(acc.NameToProperty("id"),
-                                            storage::PropertyValue(node_id.id));
-      if (!node_property.HasValue())
-        throw LoadException("Couldn't add property 'id' to the node");
-      if (!*node_property)
-        throw LoadException("The property 'id' already exists");
+      if (!field.name.empty()) {
+        storage::PropertyValue pv_id;
+        if (FLAGS_id_type == "INTEGER") {
+          pv_id = storage::PropertyValue(StringToInt(node_id.id));
+        } else {
+          pv_id = storage::PropertyValue(node_id.id);
+        }
+        auto node_property =
+            node.SetProperty(acc.NameToProperty(field.name), pv_id);
+        if (!node_property.HasValue())
+          throw LoadException("Couldn't add property '{}' to the node",
+                              field.name);
+        if (!*node_property)
+          throw LoadException("The property '{}' already exists", field.name);
+      }
       id = node_id;
     } else if (field.type == "LABEL") {
       for (const auto &label : utils::Split(value, FLAGS_array_delimiter)) {
-        auto node_label = node.AddLabel(acc.NameToLabel(utils::Trim(label)));
+        auto node_label = node.AddLabel(acc.NameToLabel(label));
         if (!node_label.HasValue())
-          throw LoadException("Couldn't add label '{}' to the node",
-                              utils::Trim(label));
+          throw LoadException("Couldn't add label '{}' to the node", label);
         if (!*node_label)
-          throw LoadException("The label '{}' already exists",
-                              utils::Trim(label));
+          throw LoadException("The label '{}' already exists", label);
       }
     } else if (field.type != "IGNORE") {
       auto node_property = node.SetProperty(acc.NameToProperty(field.name),
@@ -436,35 +488,41 @@ void ProcessNodeRow(storage::Storage *store, const std::vector<Field> &fields,
     }
   }
   for (const auto &label : additional_labels) {
-    auto node_label = node.AddLabel(acc.NameToLabel(utils::Trim(label)));
+    auto node_label = node.AddLabel(acc.NameToLabel(label));
     if (!node_label.HasValue())
-      throw LoadException("Couldn't add label '{}' to the node",
-                          utils::Trim(label));
+      throw LoadException("Couldn't add label '{}' to the node", label);
     if (!*node_label)
-      throw LoadException("The label '{}' already exists", utils::Trim(label));
+      throw LoadException("The label '{}' already exists", label);
   }
-  if (!id) throw LoadException("Node ID must be specified");
   if (acc.Commit().HasError()) throw LoadException("Couldn't store the node");
 }
 
 void ProcessNodes(storage::Storage *store, const std::string &nodes_path,
+                  std::optional<std::vector<Field>> *header,
                   std::unordered_map<NodeId, storage::Gid> *node_id_map,
                   const std::vector<std::string> &additional_labels) {
   std::ifstream nodes_file(nodes_path);
   CHECK(nodes_file) << "Unable to open '" << nodes_path << "'";
   uint64_t row_number = 1;
   try {
-    auto [fields, header_lines] = ReadHeader(nodes_file);
-    row_number += header_lines;
+    if (!*header) {
+      auto [fields, header_lines] = ReadHeader(nodes_file);
+      row_number += header_lines;
+      header->emplace(std::move(fields));
+    }
     while (true) {
       auto [row, lines_count] = ReadRow(nodes_file);
       if (lines_count == 0) break;
-      if (row.size() != fields.size())
+      if ((!FLAGS_ignore_extra_columns && row.size() != (*header)->size()) ||
+          (FLAGS_ignore_extra_columns && row.size() < (*header)->size()))
         throw LoadException(
             "Expected as many values as there are header fields (found {}, "
             "expected {})",
-            row.size(), fields.size());
-      ProcessNodeRow(store, fields, row, additional_labels, node_id_map);
+            row.size(), (*header)->size());
+      if (row.size() > (*header)->size()) {
+        row.resize((*header)->size());
+      }
+      ProcessNodeRow(store, **header, row, additional_labels, node_id_map);
       row_number += lines_count;
     }
   } catch (const LoadException &e) {
@@ -477,40 +535,61 @@ void ProcessNodes(storage::Storage *store, const std::string &nodes_path,
 void ProcessRelationshipsRow(
     storage::Storage *store, const std::vector<Field> &fields,
     const std::vector<std::string> &row,
+    std::optional<std::string> relationship_type,
     const std::unordered_map<NodeId, storage::Gid> &node_id_map) {
   std::optional<storage::Gid> start_id;
   std::optional<storage::Gid> end_id;
-  std::optional<std::string> relationship_type;
   std::map<std::string, storage::PropertyValue> properties;
   for (size_t i = 0; i < row.size(); ++i) {
     const auto &field = fields[i];
-    std::string value(utils::Trim(row[i]));
+    const auto &value = row[i];
     if (utils::StartsWith(field.type, "START_ID")) {
       if (start_id) throw LoadException("Only one node ID must be specified");
+      if (FLAGS_id_type == "INTEGER") {
+        // Call `StringToInt` to verify that the START_ID is a valid integer.
+        StringToInt(value);
+      }
       NodeId node_id{value, GetIdSpace(field.type)};
       auto it = node_id_map.find(node_id);
-      if (it == node_id_map.end())
-        throw LoadException("Node with id '{}' does not exist", node_id);
+      if (it == node_id_map.end()) {
+        if (FLAGS_skip_bad_relationships) {
+          LOG(WARNING) << "Skipping bad relationship with START_ID '" << node_id
+                       << "'";
+          return;
+        } else {
+          throw LoadException("Node with ID '{}' does not exist", node_id);
+        }
+      }
       start_id = it->second;
     } else if (utils::StartsWith(field.type, "END_ID")) {
       if (end_id) throw LoadException("Only one node ID must be specified");
+      if (FLAGS_id_type == "INTEGER") {
+        // Call `StringToInt` to verify that the END_ID is a valid integer.
+        StringToInt(value);
+      }
       NodeId node_id{value, GetIdSpace(field.type)};
       auto it = node_id_map.find(node_id);
-      if (it == node_id_map.end())
-        throw LoadException("Node with id '{}' does not exist", node_id);
+      if (it == node_id_map.end()) {
+        if (FLAGS_skip_bad_relationships) {
+          LOG(WARNING) << "Skipping bad relationship with END_ID '" << node_id
+                       << "'";
+          return;
+        } else {
+          throw LoadException("Node with ID '{}' does not exist", node_id);
+        }
+      }
       end_id = it->second;
     } else if (field.type == "TYPE") {
       if (relationship_type)
         throw LoadException("Only one relationship TYPE must be specified");
       relationship_type = value;
     } else if (field.type != "IGNORE") {
-      properties[field.name] = StringToValue(value, field.type);
+      auto [it, inserted] =
+          properties.emplace(field.name, StringToValue(value, field.type));
+      if (!inserted)
+        throw LoadException("The property '{}' already exists", field.name);
     }
   }
-  auto rel_type = utils::Trim(FLAGS_relationship_type);
-  if (!rel_type.empty()) {
-    relationship_type = rel_type;
-  }
   if (!start_id) throw LoadException("START_ID must be set");
   if (!end_id) throw LoadException("END_ID must be set");
   if (!relationship_type) throw LoadException("Relationship TYPE must be set");
@@ -526,28 +605,54 @@ void ProcessRelationshipsRow(
   if (!relationship.HasValue())
     throw LoadException("Couldn't create the relationship");
 
+  for (const auto &property : properties) {
+    auto ret = relationship->SetProperty(acc.NameToProperty(property.first),
+                                         property.second);
+    if (!ret.HasValue()) {
+      if (ret.GetError() != storage::Error::PROPERTIES_DISABLED) {
+        throw LoadException("Couldn't add property '{}' to the relationship",
+                            property.first);
+      } else {
+        throw LoadException(
+            "Couldn't add property '{}' to the relationship because properties "
+            "on edges are disabled",
+            property.first);
+      }
+    }
+  }
+
   if (acc.Commit().HasError())
     throw LoadException("Couldn't store the relationship");
 }
 
 void ProcessRelationships(
     storage::Storage *store, const std::string &relationships_path,
+    const std::optional<std::string> &relationship_type,
+    std::optional<std::vector<Field>> *header,
     const std::unordered_map<NodeId, storage::Gid> &node_id_map) {
   std::ifstream relationships_file(relationships_path);
   CHECK(relationships_file) << "Unable to open '" << relationships_path << "'";
   uint64_t row_number = 1;
   try {
-    auto [fields, header_lines] = ReadHeader(relationships_file);
-    row_number += header_lines;
+    if (!*header) {
+      auto [fields, header_lines] = ReadHeader(relationships_file);
+      row_number += header_lines;
+      header->emplace(std::move(fields));
+    }
     while (true) {
       auto [row, lines_count] = ReadRow(relationships_file);
       if (lines_count == 0) break;
-      if (row.size() != fields.size())
+      if ((!FLAGS_ignore_extra_columns && row.size() != (*header)->size()) ||
+          (FLAGS_ignore_extra_columns && row.size() < (*header)->size()))
         throw LoadException(
             "Expected as many values as there are header fields (found {}, "
             "expected {})",
-            row.size(), fields.size());
-      ProcessRelationshipsRow(store, fields, row, node_id_map);
+            row.size(), (*header)->size());
+      if (row.size() > (*header)->size()) {
+        row.resize((*header)->size());
+      }
+      ProcessRelationshipsRow(store, **header, row, relationship_type,
+                              node_id_map);
       row_number += lines_count;
     }
   } catch (const LoadException &e) {
@@ -556,17 +661,65 @@ void ProcessRelationships(
   }
 }
 
-static const char *usage =
-    "[OPTION]... [--out=SNAPSHOT_FILE] [--nodes=CSV_FILE]... "
-    "[--relationships=CSV_FILE]...\n"
-    "Create a Memgraph recovery snapshot file from CSV.\n";
+struct NodesArgument {
+  // List of all files that have should be processed for nodes.
+  std::vector<std::string> nodes;
+  // List of all additional labels that should be added to the nodes.
+  std::vector<std::string> additional_labels;
+};
+
+NodesArgument ParseNodesArgument(const std::string &value) {
+  // The format of this argument is as follows:
+  // [<label>[:<label>]...=]<file>[,<file>][,<file>]...
+
+  std::vector<std::string> nodes;
+  std::vector<std::string> additional_labels;
+
+  size_t pos_nodes = 0;
+  auto pos_equal = value.find('=');
+  if (pos_equal != std::string::npos) {
+    // We have additional labels.
+    additional_labels = utils::Split(value.substr(0, pos_equal), ":");
+    pos_nodes = pos_equal + 1;
+  }
+
+  nodes = utils::Split(value.substr(pos_nodes), ",");
+
+  return {std::move(nodes), std::move(additional_labels)};
+}
+
+struct RelationshipsArgument {
+  // List of all files that have should be processed for relationships.
+  std::vector<std::string> relationships;
+  // Optional type of the relationships.
+  std::optional<std::string> type;
+};
+
+RelationshipsArgument ParseRelationshipsArgument(const std::string &value) {
+  // The format of this argument is as follows:
+  // [<type>=]<file>[,<file>][,<file>]...
+
+  std::vector<std::string> relationships;
+  std::optional<std::string> type;
+
+  size_t pos_relationships = 0;
+  auto pos_equal = value.find('=');
+  if (pos_equal != std::string::npos) {
+    // The type has been specified.
+    type = value.substr(0, pos_equal);
+    pos_relationships = pos_equal + 1;
+  }
+
+  relationships = utils::Split(value.substr(pos_relationships), ",");
+
+  return {std::move(relationships), std::move(type)};
+}
 
 int main(int argc, char *argv[]) {
-  gflags::SetUsageMessage(usage);
+  gflags::SetUsageMessage("Create a Memgraph recovery snapshot file from CSV.");
   gflags::SetVersionString(version_string);
 
   auto nodes = ParseRepeatedFlag("nodes", argc, argv);
-  auto additional_labels = ParseRepeatedFlag("node-label", argc, argv);
   auto relationships = ParseRepeatedFlag("relationships", argc, argv);
 
   // Load config before parsing arguments, so that flags from the command line
@@ -577,17 +730,15 @@ int main(int argc, char *argv[]) {
 
   CHECK(!nodes.empty()) << "The --nodes flag is required!";
 
+  {
+    std::string upper = utils::ToUpperCase(utils::Trim(FLAGS_id_type));
+    FLAGS_id_type = upper;
+  }
+
   // Verify that the user that started the Memgraph process is the same user
   // that is the owner of the data directory.
   VerifyDataDirectoryOwnerAndProcessUser(FLAGS_data_directory);
 
-  {
-    auto all_inputs = nodes;
-    all_inputs.insert(all_inputs.end(), relationships.begin(),
-                      relationships.end());
-    LOG(INFO) << "Loading " << utils::Join(all_inputs, ", ");
-  }
-
   std::unordered_map<NodeId, storage::Gid> node_id_map;
   storage::Storage store{
       {.durability =
@@ -603,13 +754,25 @@ int main(int argc, char *argv[]) {
   utils::Timer load_timer;
 
   // Process all nodes files.
-  for (const auto &nodes_file : nodes) {
-    ProcessNodes(&store, nodes_file, &node_id_map, additional_labels);
+  for (const auto &value : nodes) {
+    auto [files, additional_labels] = ParseNodesArgument(value);
+    std::optional<std::vector<Field>> header;
+    for (const auto &nodes_file : files) {
+      LOG(INFO) << "Loading " << nodes_file;
+      ProcessNodes(&store, nodes_file, &header, &node_id_map,
+                   additional_labels);
+    }
   }
 
   // Process all relationships files.
-  for (const auto &relationships_file : relationships) {
-    ProcessRelationships(&store, relationships_file, node_id_map);
+  for (const auto &value : relationships) {
+    auto [files, type] = ParseRelationshipsArgument(value);
+    std::optional<std::vector<Field>> header;
+    for (const auto &relationships_file : files) {
+      LOG(INFO) << "Loading " << relationships_file;
+      ProcessRelationships(&store, relationships_file, type, &header,
+                           node_id_map);
+    }
   }
 
   double load_sec = load_timer.Elapsed().count();

src/utils/string.hpp

@@ -313,6 +313,33 @@ inline std::vector<std::string> RSplit(const std::string_view &src,
   return res;
 }
 
+/**
+ * Parse a signed integer value from a string using classic locale.
+ * Note, the current implementation copies the given string which may perform a
+ * heap allocation if the string is big enough.
+ *
+ * @throw BasicException if unable to parse the whole string.
+ */
+inline int64_t ParseInt(const std::string_view &s) {
+  // stol would be nicer but it uses current locale so we shouldn't use it.
+  int64_t t = 0;
+  // NOTE: Constructing std::istringstream will make a copy of the string, which
+  // may make a heap allocation if string is large enough. There is no
+  // std::istringstream constructor accepting a custom allocator. We could pass
+  // a std::basic_string with a custom allocator, but std::istringstream will
+  // probably invoke
+  // std::allocator_traits<>::select_on_container_copy_construction which
+  // doesn't really help as most allocators default to global new/delete
+  // allocator.
+  std::istringstream iss(std::string(s.data(), s.size()));
+  iss.imbue(std::locale::classic());
+  iss >> t;
+  if (iss.fail() || !iss.eof()) {
+    throw BasicException("Couldn't parse string");
+  }
+  return t;
+}
+
 /**
  * Parse a double floating point value from a string using classic locale.
  * Note, the current implementation copies the given string which may perform a

tests/integration/mg_import_csv/tests/array_types/expected.cypher

@@ -0,0 +1,10 @@
+CREATE INDEX ON :__mg_vertex__(__mg_id__);
+CREATE (:__mg_vertex__:Message:Comment:Entry0:hello {__mg_id__: 0, country: "Croatia", content: "yes", browser: "Chrome"});
+CREATE (:__mg_vertex__:Message:Comment:Entry1 {__mg_id__: 1, country: "United Kingdom", content: "thanks", browser: "Chrome"});
+CREATE (:__mg_vertex__:Message:Comment:Entry2:world {__mg_id__: 2, country: "Germany", content: "LOL"});
+CREATE (:__mg_vertex__:Message:Comment:Entry3 {__mg_id__: 3, country: "France", content: "I see", browser: "Firefox"});
+CREATE (:__mg_vertex__:Message:Comment:Entry4:this:is:a: lot :of: labels {__mg_id__: 4, country: "Italy", content: "fine", browser: "Internet Explorer"});
+MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 1 AND v.__mg_id__ = 2 CREATE (u)-[:KNOWS {value: ["5", " asd", " helloworld"]}]->(v);
+MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 4 AND v.__mg_id__ = 0 CREATE (u)-[:KNOWS {value: ["6", "hmmm"]}]->(v);
+DROP INDEX ON :__mg_vertex__(__mg_id__);
+MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;

tests/integration/mg_import_csv/tests/array_types/nodes.csv

@@ -0,0 +1,6 @@
+:ID,country:string,browser:string,content:string,:LABEL
+0,Croatia,Chrome,yes,Message;Comment;Entry0;hello
+1,"United Kingdom",Chrome,thanks,Message;Comment;Entry1
+2,Germany,,LOL,Message;Comment;Entry2;world
+3,France,Firefox,I see,Message;Comment;Entry3
+4,Italy,Internet Explorer,fine,Message;Comment;Entry4;this;is;a; lot ;of; labels

tests/integration/mg_import_csv/tests/array_types/relationships.csv

@@ -0,0 +1,3 @@
+:START_ID,:END_ID,:TYPE,value:string[]
+1,2,KNOWS,5; asd; helloworld
+4,0,KNOWS,6;hmmm

tests/integration/mg_import_csv/tests/array_types/test.yaml

@@ -0,0 +1,13 @@
+- name: good_configuration
+  nodes: "nodes.csv"
+  relationships: "relationships.csv"
+  properties_on_edges: True
+  ignore_empty_strings: True
+  expected: expected.cypher
+
+- name: properties_on_edges_disabled
+  nodes: "nodes.csv"
+  relationships: "relationships.csv"
+  properties_on_edges: False
+  ignore_empty_strings: True
+  import_should_fail: True

tests/integration/mg_import_csv/tests/array_types_multi_char_array_delimiter/expected.cypher

@@ -0,0 +1,10 @@
+CREATE INDEX ON :__mg_vertex__(__mg_id__);
+CREATE (:__mg_vertex__:Message:Comment:Entry0:hello {__mg_id__: 0, country: "Croatia", content: "yes", browser: "Chrome"});
+CREATE (:__mg_vertex__:Message:Comment:Entry1 {__mg_id__: 1, country: "United Kingdom", content: "thanks", browser: "Chrome"});
+CREATE (:__mg_vertex__:Message:Comment:Entry2:world {__mg_id__: 2, country: "Germany", content: "LOL"});
+CREATE (:__mg_vertex__:Message:Comment:Entry3 {__mg_id__: 3, country: "France", content: "I see", browser: "Firefox"});
+CREATE (:__mg_vertex__:Message:Comment:Entry4:this:is:a: lot :of: labels {__mg_id__: 4, country: "Italy", content: "fine", browser: "Internet Explorer"});
+MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 1 AND v.__mg_id__ = 2 CREATE (u)-[:KNOWS {value: ["5", " asd", " helloworld"]}]->(v);
+MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 4 AND v.__mg_id__ = 0 CREATE (u)-[:KNOWS {value: ["6", "hmmm"]}]->(v);
+DROP INDEX ON :__mg_vertex__(__mg_id__);
+MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;

tests/integration/mg_import_csv/tests/array_types_multi_char_array_delimiter/nodes.csv

@@ -0,0 +1,6 @@
+:ID,country:string,browser:string,content:string,:LABEL
+0,Croatia,Chrome,yes,MessageÖCommentÖEntry0Öhello
+1,"United Kingdom",Chrome,thanks,MessageÖCommentÖEntry1
+2,Germany,,LOL,MessageÖCommentÖEntry2Öworld
+3,France,Firefox,I see,MessageÖCommentÖEntry3
+4,Italy,Internet Explorer,fine,MessageÖCommentÖEntry4ÖthisÖisÖaÖ lot ÖofÖ labels

tests/integration/mg_import_csv/tests/array_types_multi_char_array_delimiter/relationships.csv

@@ -0,0 +1,3 @@
+:START_ID,:END_ID,:TYPE,value:string[]
+1,2,KNOWS,5Ö asdÖ helloworld
+4,0,KNOWS,6Öhmmm

tests/integration/mg_import_csv/tests/array_types_multi_char_array_delimiter/test.yaml

@@ -0,0 +1,15 @@
+- name: good_configuration
+  nodes: "nodes.csv"
+  relationships: "relationships.csv"
+  array_delimiter: "Ö"
+  properties_on_edges: True
+  ignore_empty_strings: True
+  expected: expected.cypher
+
+- name: properties_on_edges_disabled
+  nodes: "nodes.csv"
+  relationships: "relationships.csv"
+  array_delimiter: "Ö"
+  properties_on_edges: False
+  ignore_empty_strings: True
+  import_should_fail: True

tests/integration/mg_import_csv/tests/bad_relationships/expected.cypher

@@ -0,0 +1,11 @@
+CREATE INDEX ON :__mg_vertex__(__mg_id__);
+CREATE (:__mg_vertex__:Message:Comment {__mg_id__: 0, id: "0", country: "Croatia", browser: "Chrome", content: "yes"});
+CREATE (:__mg_vertex__:Message:Comment {__mg_id__: 1, id: "1", country: "United Kingdom", browser: "Chrome", content: "thanks"});
+CREATE (:__mg_vertex__:Message:Comment {__mg_id__: 2, id: "2", country: "Germany", content: "LOL"});
+CREATE (:__mg_vertex__:Message:Comment {__mg_id__: 3, id: "3", country: "France", browser: "Firefox", content: "I see"});
+CREATE (:__mg_vertex__:Message:Comment {__mg_id__: 4, id: "4", country: "Italy", browser: "Internet Explorer", content: "fine"});
+MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 0 AND v.__mg_id__ = 1 CREATE (u)-[:LIKES]->(v);
+MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 0 AND v.__mg_id__ = 2 CREATE (u)-[:VISITED]->(v);
+MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 1 AND v.__mg_id__ = 2 CREATE (u)-[:FOLLOWS]->(v);
+DROP INDEX ON :__mg_vertex__(__mg_id__);
+MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;

tests/integration/mg_import_csv/tests/bad_relationships/nodes.csv

@@ -0,0 +1,6 @@
+id:ID(COMMENT_ID),country:string,browser:string,content:string,:LABEL
+0,Croatia,Chrome,yes,Message;Comment
+1,"United Kingdom",Chrome,thanks,Message;Comment
+2,Germany,,LOL,Message;Comment
+3,France,Firefox,I see,Message;Comment
+4,Italy,Internet Explorer,fine,Message;Comment

tests/integration/mg_import_csv/tests/bad_relationships/relationships.csv

@@ -0,0 +1,7 @@
+:START_ID(COMMENT_ID),:END_ID(COMMENT_ID),:TYPE
+0,1,LIKES
+1,2,FOLLOWS
+0,5,INVALID_END
+5,0,INVALID_START
+6,5,INVALID_BOTH
+0,2,VISITED

tests/integration/mg_import_csv/tests/bad_relationships/test.yaml

@@ -0,0 +1,12 @@
+- name: good_configuration
+  nodes: "nodes.csv"
+  relationships: "relationships.csv"
+  ignore_empty_strings: True
+  skip_bad_relationships: True
+  expected: expected.cypher
+
+- name: missing_skip_bad_relationships
+  nodes: "nodes.csv"
+  relationships: "relationships.csv"
+  ignore_empty_strings: True
+  import_should_fail: True

tests/integration/mg_import_csv/tests/csv_parser_test1/nodes.csv

@@ -1,4 +1,4 @@
-:ID,value
+id:ID,value
 0,
 1,""
 2,hello

tests/integration/mg_import_csv/tests/csv_parser_test1/test.yaml

@@ -1,8 +1,10 @@
 - name: good_configuration
   nodes: "nodes.csv"
+  ignore_empty_strings: True
   expected: expected.cypher
 
 - name: wrong_delimiter
   nodes: "nodes.csv"
+  ignore_empty_strings: True
   delimiter: "-"
   import_should_fail: True

tests/integration/mg_import_csv/tests/csv_parser_test10/nodes.csv

@@ -1,4 +1,4 @@
-:ID,value,test
+id:ID,value,test
 0,Öworld
 
 

tests/integration/mg_import_csv/tests/csv_parser_test10/test.yaml

@@ -1,15 +1,18 @@
 - name: good_configuration
   nodes: "nodes.csv"
+  ignore_empty_strings: True
   quote: "Ö"
   expected: expected.cypher
 
 - name: wrong_delimiter
   nodes: "nodes.csv"
+  ignore_empty_strings: True
   quote: "Ö"
   delimiter: "-"
   import_should_fail: True
 
 - name: wrong_quote
   nodes: "nodes.csv"
+  ignore_empty_strings: True
   quote: "-"
   import_should_fail: True

tests/integration/mg_import_csv/tests/csv_parser_test11/nodes.csv

@@ -1,4 +1,4 @@
-:ID,value,test
+id:ID,value,test
 0,Öworld
 
 

tests/integration/mg_import_csv/tests/csv_parser_test12/nodes.csv

@@ -1,4 +1,4 @@
-:ID,value,test
+id:ID,value,test
 0,ÖÖÖworld
 
 

tests/integration/mg_import_csv/tests/csv_parser_test12/test.yaml

@@ -1,15 +1,18 @@
 - name: good_configuration
   nodes: "nodes.csv"
+  ignore_empty_strings: True
   quote: "Ö"
   expected: expected.cypher
 
 - name: wrong_delimiter
   nodes: "nodes.csv"
+  ignore_empty_strings: True
   quote: "Ö"
   delimiter: "-"
   import_should_fail: True
 
 - name: wrong_quote
   nodes: "nodes.csv"
+  ignore_empty_strings: True
   quote: "-"
   import_should_fail: True

tests/integration/mg_import_csv/tests/csv_parser_test13/nodes.csv

@@ -1,4 +1,4 @@
-:ID,value,test
+id:ID,value,test
 0,ÖÖÖworld,
 
 

tests/integration/mg_import_csv/tests/csv_parser_test13/test.yaml

@@ -1,15 +1,18 @@
 - name: good_configuration
   nodes: "nodes.csv"
+  ignore_empty_strings: True
   quote: "Ö"
   expected: expected.cypher
 
 - name: wrong_delimiter
   nodes: "nodes.csv"
+  ignore_empty_strings: True
   quote: "Ö"
   delimiter: "-"
   import_should_fail: True
 
 - name: wrong_quote
   nodes: "nodes.csv"
+  ignore_empty_strings: True
   quote: "-"
   import_should_fail: True

tests/integration/mg_import_csv/tests/csv_parser_test14/nodes.csv

@@ -1,4 +1,4 @@
-:ID,value,test
+id:ID,value,test
 0,Öworld
 
 

tests/integration/mg_import_csv/tests/csv_parser_test15/nodes.csv

@@ -1,4 +1,4 @@
-:ID,value,test
+id:ID,value,test
 0,woÖrld,asd
 1,,string
 2,hello,wilÖl this work?ÖÖ

tests/integration/mg_import_csv/tests/csv_parser_test15/test.yaml

@@ -1,15 +1,18 @@
 - name: good_configuration
   nodes: "nodes.csv"
+  ignore_empty_strings: True
   quote: "Ö"
   expected: expected.cypher
 
 - name: wrong_delimiter
   nodes: "nodes.csv"
+  ignore_empty_strings: True
   quote: "Ö"
   delimiter: "-"
   import_should_fail: True
 
 - name: wrong_quote
   nodes: "nodes.csv"
+  ignore_empty_strings: True
   quote: "-"
   expected: expected.cypher

tests/integration/mg_import_csv/tests/csv_parser_test16/nodes.csv

@@ -1,4 +1,4 @@
-:IDÖvalue
+id:IDÖvalue
 0Ö
 1Ö""
 2Öhello

tests/integration/mg_import_csv/tests/csv_parser_test16/test.yaml

@@ -1,9 +1,11 @@
 - name: good_configuration
   nodes: "nodes.csv"
+  ignore_empty_strings: True
   delimiter: "Ö"
   expected: expected.cypher
 
 - name: wrong_delimiter
   nodes: "nodes.csv"
+  ignore_empty_strings: True
   delimiter: "-"
   import_should_fail: True

tests/integration/mg_import_csv/tests/csv_parser_test17/nodes.csv

@@ -1,4 +1,4 @@
-:IDÖvalueÖtest
+id:IDÖvalueÖtest
 0ÖworldÖasd
 1ÖÖstring
 2ÖhelloÖmy

tests/integration/mg_import_csv/tests/csv_parser_test17/test.yaml

@@ -1,9 +1,11 @@
 - name: good_configuration
   nodes: "nodes.csv"
+  ignore_empty_strings: True
   delimiter: "Ö"
   expected: expected.cypher
 
 - name: wrong_delimiter
   nodes: "nodes.csv"
+  ignore_empty_strings: True
   delimiter: "-"
   import_should_fail: True

tests/integration/mg_import_csv/tests/csv_parser_test18/nodes.csv

@@ -1,4 +1,4 @@
-:IDÖvalueÖtest
+id:IDÖvalueÖtest
 0Ö"world
 
 

tests/integration/mg_import_csv/tests/csv_parser_test18/test.yaml

@@ -1,15 +1,18 @@
 - name: good_configuration
   nodes: "nodes.csv"
+  ignore_empty_strings: True
   delimiter: "Ö"
   expected: expected.cypher
 
 - name: wrong_delimiter
   nodes: "nodes.csv"
+  ignore_empty_strings: True
   delimiter: "-"
   import_should_fail: True
 
 - name: wrong_quote
   nodes: "nodes.csv"
+  ignore_empty_strings: True
   delimiter: "Ö"
   quote: "-"
   import_should_fail: True

tests/integration/mg_import_csv/tests/csv_parser_test19/nodes.csv

@@ -1,4 +1,4 @@
-:IDÖvalueÖtest
+id:IDÖvalueÖtest
 0Ö"world
 
 

tests/integration/mg_import_csv/tests/csv_parser_test2/nodes.csv

@@ -1,4 +1,4 @@
-:ID,value,test
+id:ID,value,test
 0,world,asd
 1,,string
 2,hello,my

tests/integration/mg_import_csv/tests/csv_parser_test2/test.yaml

@@ -1,8 +1,10 @@
 - name: good_configuration
   nodes: "nodes.csv"
+  ignore_empty_strings: True
   expected: expected.cypher
 
 - name: wrong_delimiter
   nodes: "nodes.csv"
+  ignore_empty_strings: True
   delimiter: "-"
   import_should_fail: True

tests/integration/mg_import_csv/tests/csv_parser_test20/nodes.csv

@@ -1,4 +1,4 @@
-:IDÖvalueÖtest
+id:IDÖvalueÖtest
 0Ö"""world
 
 

tests/integration/mg_import_csv/tests/csv_parser_test20/test.yaml

@@ -1,15 +1,18 @@
 - name: good_configuration
   nodes: "nodes.csv"
+  ignore_empty_strings: True
   delimiter: "Ö"
   expected: expected.cypher
 
 - name: wrong_delimiter
   nodes: "nodes.csv"
+  ignore_empty_strings: True
   delimiter: "-"
   import_should_fail: True
 
 - name: wrong_quote
   nodes: "nodes.csv"
+  ignore_empty_strings: True
   delimiter: "Ö"
   quote: "-"
   import_should_fail: True

tests/integration/mg_import_csv/tests/csv_parser_test21/nodes.csv

@@ -1,4 +1,4 @@
-:IDÖvalueÖtest
+id:IDÖvalueÖtest
 0Ö"""worldÖ
 
 

tests/integration/mg_import_csv/tests/csv_parser_test21/test.yaml

@@ -1,15 +1,18 @@
 - name: good_configuration
   nodes: "nodes.csv"
+  ignore_empty_strings: True
   delimiter: "Ö"
   expected: expected.cypher
 
 - name: wrong_delimiter
   nodes: "nodes.csv"
+  ignore_empty_strings: True
   delimiter: "-"
   import_should_fail: True
 
 - name: wrong_quote
   nodes: "nodes.csv"
+  ignore_empty_strings: True
   delimiter: "Ö"
   quote: "-"
   import_should_fail: True

tests/integration/mg_import_csv/tests/csv_parser_test22/nodes.csv

@@ -1,4 +1,4 @@
-:IDÖvalueÖtest
+id:IDÖvalueÖtest
 0Ö"world
 
 

tests/integration/mg_import_csv/tests/csv_parser_test23/nodes.csv

@@ -1,4 +1,4 @@
-:IDÖvalueÖtest
+id:IDÖvalueÖtest
 0Öwo"rldÖasd
 1ÖÖstring
 2ÖhelloÖwil"l this work?""

tests/integration/mg_import_csv/tests/csv_parser_test23/test.yaml

@@ -1,15 +1,18 @@
 - name: good_configuration
   nodes: "nodes.csv"
+  ignore_empty_strings: True
   delimiter: "Ö"
   expected: expected.cypher
 
 - name: wrong_delimiter
   nodes: "nodes.csv"
+  ignore_empty_strings: True
   delimiter: "-"
   import_should_fail: True
 
 - name: wrong_quote
   nodes: "nodes.csv"
+  ignore_empty_strings: True
   delimiter: "Ö"
   quote: "-"
   expected: expected.cypher

tests/integration/mg_import_csv/tests/csv_parser_test3/nodes.csv

@@ -1,4 +1,4 @@
-:ID,value,test
+id:ID,value,test
 0,"world
 
 

tests/integration/mg_import_csv/tests/csv_parser_test3/test.yaml

@@ -1,13 +1,16 @@
 - name: good_configuration
   nodes: "nodes.csv"
+  ignore_empty_strings: True
   expected: expected.cypher
 
 - name: wrong_delimiter
   nodes: "nodes.csv"
+  ignore_empty_strings: True
   delimiter: "-"
   import_should_fail: True
 
 - name: wrong_quote
   nodes: "nodes.csv"
+  ignore_empty_strings: True
   quote: "-"
   import_should_fail: True

tests/integration/mg_import_csv/tests/csv_parser_test4/nodes.csv

@@ -1,4 +1,4 @@
-:ID,value,test
+id:ID,value,test
 0,"world
 
 

tests/integration/mg_import_csv/tests/csv_parser_test5/nodes.csv

@@ -1,4 +1,4 @@
-:ID,value,test
+id:ID,value,test
 0,"""world
 
 

tests/integration/mg_import_csv/tests/csv_parser_test5/test.yaml

@@ -1,13 +1,16 @@
 - name: good_configuration
   nodes: "nodes.csv"
+  ignore_empty_strings: True
   expected: expected.cypher
 
 - name: wrong_delimiter
   nodes: "nodes.csv"
+  ignore_empty_strings: True
   delimiter: "-"
   import_should_fail: True
 
 - name: wrong_quote
   nodes: "nodes.csv"
+  ignore_empty_strings: True
   quote: "-"
   import_should_fail: True

tests/integration/mg_import_csv/tests/csv_parser_test6/nodes.csv

@@ -1,4 +1,4 @@
-:ID,value,test
+id:ID,value,test
 0,"""world,
 
 

tests/integration/mg_import_csv/tests/csv_parser_test6/test.yaml

@@ -1,13 +1,16 @@
 - name: good_configuration
   nodes: "nodes.csv"
+  ignore_empty_strings: True
   expected: expected.cypher
 
 - name: wrong_delimiter
   nodes: "nodes.csv"
+  ignore_empty_strings: True
   delimiter: "-"
   import_should_fail: True
 
 - name: wrong_quote
   nodes: "nodes.csv"
+  ignore_empty_strings: True
   quote: "-"
   import_should_fail: True

tests/integration/mg_import_csv/tests/csv_parser_test7/nodes.csv

@@ -1,4 +1,4 @@
-:ID,value,test
+id:ID,value,test
 0,"world
 
 

tests/integration/mg_import_csv/tests/csv_parser_test8/nodes.csv

@@ -1,4 +1,4 @@
-:ID,value,test
+id:ID,value,test
 0,wo"rld,asd
 1,,string
 2,hello,wil"l this work?""

tests/integration/mg_import_csv/tests/csv_parser_test8/test.yaml

@@ -1,13 +1,16 @@
 - name: good_configuration
   nodes: "nodes.csv"
+  ignore_empty_strings: True
   expected: expected.cypher
 
 - name: wrong_delimiter
   nodes: "nodes.csv"
+  ignore_empty_strings: True
   delimiter: "-"
   import_should_fail: True
 
 - name: wrong_quote
   nodes: "nodes.csv"
+  ignore_empty_strings: True
   quote: "-"
   expected: expected.cypher

tests/integration/mg_import_csv/tests/csv_parser_test9/nodes.csv

@@ -1,4 +1,4 @@
-:ID,value
+id:ID,value
 0,
 1,ÖÖ
 2,hello

tests/integration/mg_import_csv/tests/csv_parser_test9/test.yaml

@@ -1,10 +1,12 @@
 - name: good_configuration
   nodes: "nodes.csv"
+  ignore_empty_strings: True
   quote: "Ö"
   expected: expected.cypher
 
 - name: wrong_delimiter
   nodes: "nodes.csv"
+  ignore_empty_strings: True
   quote: "Ö"
   delimiter: "-"
   import_should_fail: True

tests/integration/mg_import_csv/tests/data_split_into_multiple_files/expected.cypher

@@ -0,0 +1,17 @@
+CREATE INDEX ON :__mg_vertex__(__mg_id__);
+CREATE (:__mg_vertex__ {__mg_id__: 0, country: "Austria", value: 6});
+CREATE (:__mg_vertex__ {__mg_id__: 1, country: "Hungary", value: 7});
+CREATE (:__mg_vertex__ {__mg_id__: 2, country: "Romania", value: 8});
+CREATE (:__mg_vertex__ {__mg_id__: 3, country: "Bulgaria", value: 9});
+CREATE (:__mg_vertex__ {__mg_id__: 4, country: "Spain", value: 10});
+CREATE (:__mg_vertex__ {__mg_id__: 5, country: "Latvia", value: 11});
+CREATE (:__mg_vertex__ {__mg_id__: 6, country: "Russia", value: 12});
+CREATE (:__mg_vertex__ {__mg_id__: 7, country: "Poland", value: 13});
+CREATE (:__mg_vertex__ {__mg_id__: 8, country: "Czech Republic", value: 14});
+CREATE (:__mg_vertex__ {__mg_id__: 9, country: "Moldova", value: 15});
+MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 0 AND v.__mg_id__ = 1 CREATE (u)-[:NEIGHBOUR]->(v);
+MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 2 AND v.__mg_id__ = 1 CREATE (u)-[:NEIGHBOUR]->(v);
+MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 4 AND v.__mg_id__ = 6 CREATE (u)-[:NOT_NEIGHBOUR]->(v);
+MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 5 AND v.__mg_id__ = 0 CREATE (u)-[:NOT_NEIGHBOUR]->(v);
+DROP INDEX ON :__mg_vertex__(__mg_id__);
+MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;

tests/integration/mg_import_csv/tests/data_split_into_multiple_files/nodes_1.csv

@@ -0,0 +1,4 @@
+:ID,country,value:int
+1,Austria,6
+2,Hungary,7
+3,Romania,8

tests/integration/mg_import_csv/tests/data_split_into_multiple_files/nodes_2.csv

@@ -0,0 +1,7 @@
+4,Bulgaria,9
+5,Spain,10
+6,Latvia,11
+7,Russia,12
+8,Poland,13
+9,"Czech Republic",14
+10,Moldova,15

tests/integration/mg_import_csv/tests/data_split_into_multiple_files/relationships_1.csv

@@ -0,0 +1 @@
+:START_ID,:TYPE,:END_ID

tests/integration/mg_import_csv/tests/data_split_into_multiple_files/relationships_2.csv

@@ -0,0 +1,4 @@
+1,NEIGHBOUR,2
+3,NEIGHBOUR,2
+5,NOT_NEIGHBOUR,7
+6,NOT_NEIGHBOUR,1

tests/integration/mg_import_csv/tests/data_split_into_multiple_files/test.yaml

@@ -0,0 +1,5 @@
+- name: multiple_files
+  nodes: "nodes_1.csv,nodes_2.csv"
+  relationships: "relationships_1.csv,relationships_2.csv"
+  id_type: "integer"
+  expected: expected.cypher

tests/integration/mg_import_csv/tests/duplicate_nodes/expected.cypher

@@ -0,0 +1,8 @@
+CREATE INDEX ON :__mg_vertex__(__mg_id__);
+CREATE (:__mg_vertex__:Message:Comment {__mg_id__: 0, content: "yes", browser: "Chrome", country: "Croatia", id: "0"});
+CREATE (:__mg_vertex__:Message:Comment {__mg_id__: 1, content: "thanks", browser: "Chrome", country: "United Kingdom", id: "1"});
+CREATE (:__mg_vertex__:Message:Comment {__mg_id__: 2, content: "LOL", country: "Germany", id: "2"});
+CREATE (:__mg_vertex__:Message:Comment {__mg_id__: 3, content: "I see", browser: "Firefox", country: "France", id: "3"});
+CREATE (:__mg_vertex__:Message:Comment {__mg_id__: 4, content: "fine", browser: "Internet Explorer", country: "Italy", id: "4"});
+DROP INDEX ON :__mg_vertex__(__mg_id__);
+MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;

tests/integration/mg_import_csv/tests/duplicate_nodes/nodes.csv

@@ -0,0 +1,7 @@
+id:ID(COMMENT_ID),country:string,browser:string,:IGNORE,content:string,:LABEL
+0,Croatia,Chrome,this,yes,Message;Comment
+1,"United Kingdom",Chrome,should,thanks,Message;Comment
+2,Germany,,not,LOL,Message;Comment
+3,France,Firefox,appear,I see,Message;Comment
+4,Italy,Internet Explorer,anywhere,fine,Message;Comment
+0,this,is,a,duplicate,node

tests/integration/mg_import_csv/tests/duplicate_nodes/test.yaml

@@ -0,0 +1,10 @@
+- name: good_configuration
+  nodes: "nodes.csv"
+  ignore_empty_strings: True
+  skip_duplicate_nodes: True
+  expected: expected.cypher
+
+- name: missing_skip_duplicate_nodes
+  nodes: "nodes.csv"
+  ignore_empty_strings: True
+  import_should_fail: True

tests/integration/mg_import_csv/tests/extra_columns_nodes/expected.cypher

@@ -0,0 +1,8 @@
+CREATE INDEX ON :__mg_vertex__(__mg_id__);
+CREATE (:__mg_vertex__:Message:Comment {__mg_id__: 0, content: "yes", browser: "Chrome", country: "Croatia", id: "0"});
+CREATE (:__mg_vertex__:Message:Comment {__mg_id__: 1, content: "thanks", browser: "Chrome", country: "United Kingdom", id: "1"});
+CREATE (:__mg_vertex__:Message:Comment {__mg_id__: 2, content: "LOL", country: "Germany", id: "2"});
+CREATE (:__mg_vertex__:Message:Comment {__mg_id__: 3, content: "I see", browser: "Firefox", country: "France", id: "3"});
+CREATE (:__mg_vertex__:Message:Comment {__mg_id__: 4, content: "fine", browser: "Internet Explorer", country: "Italy", id: "4"});
+DROP INDEX ON :__mg_vertex__(__mg_id__);
+MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;

tests/integration/mg_import_csv/tests/extra_columns_nodes/nodes.csv

@@ -0,0 +1,6 @@
+id:ID(COMMENT_ID),country:string,browser:string,content:string,:LABEL
+0,Croatia,Chrome,yes,Message;Comment,not
+1,"United Kingdom",Chrome,thanks,Message;Comment,should
+2,Germany,,LOL,Message;Comment,not
+3,France,Firefox,I see,Message;Comment,appear
+4,Italy,Internet Explorer,fine,Message;Comment,anywhere

tests/integration/mg_import_csv/tests/extra_columns_nodes/test.yaml

@@ -0,0 +1,10 @@
+- name: good_configuration
+  nodes: "nodes.csv"
+  ignore_empty_strings: True
+  ignore_extra_columns: True
+  expected: expected.cypher
+
+- name: missing_ignore_extra_columns
+  nodes: "nodes.csv"
+  ignore_empty_strings: True
+  import_should_fail: True

tests/integration/mg_import_csv/tests/extra_columns_relationships/expected.cypher

@@ -0,0 +1,11 @@
+CREATE INDEX ON :__mg_vertex__(__mg_id__);
+CREATE (:__mg_vertex__:Message:Comment {__mg_id__: 0, id: "0", country: "Croatia", browser: "Chrome", content: "yes"});
+CREATE (:__mg_vertex__:Message:Comment {__mg_id__: 1, id: "1", country: "United Kingdom", browser: "Chrome", content: "thanks"});
+CREATE (:__mg_vertex__:Message:Comment {__mg_id__: 2, id: "2", country: "Germany", content: "LOL"});
+CREATE (:__mg_vertex__:Message:Comment {__mg_id__: 3, id: "3", country: "France", browser: "Firefox", content: "I see"});
+CREATE (:__mg_vertex__:Message:Comment {__mg_id__: 4, id: "4", country: "Italy", browser: "Internet Explorer", content: "fine"});
+MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 0 AND v.__mg_id__ = 1 CREATE (u)-[:LIKES]->(v);
+MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 0 AND v.__mg_id__ = 2 CREATE (u)-[:VISITED]->(v);
+MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 1 AND v.__mg_id__ = 2 CREATE (u)-[:FOLLOWS]->(v);
+DROP INDEX ON :__mg_vertex__(__mg_id__);
+MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;

tests/integration/mg_import_csv/tests/extra_columns_relationships/nodes.csv

@@ -0,0 +1,6 @@
+id:ID(COMMENT_ID),country:string,browser:string,content:string,:LABEL
+0,Croatia,Chrome,yes,Message;Comment
+1,"United Kingdom",Chrome,thanks,Message;Comment
+2,Germany,,LOL,Message;Comment
+3,France,Firefox,I see,Message;Comment
+4,Italy,Internet Explorer,fine,Message;Comment

tests/integration/mg_import_csv/tests/extra_columns_relationships/relationships.csv

@@ -0,0 +1,4 @@
+:START_ID(COMMENT_ID),:END_ID(COMMENT_ID),:TYPE
+0,1,LIKES,this
+1,2,FOLLOWS,is
+0,2,VISITED,more,columns,than,expected

tests/integration/mg_import_csv/tests/extra_columns_relationships/test.yaml

@@ -0,0 +1,12 @@
+- name: good_configuration
+  nodes: "nodes.csv"
+  relationships: "relationships.csv"
+  ignore_empty_strings: True
+  ignore_extra_columns: True
+  expected: expected.cypher
+
+- name: missing_ignore_extra_columns
+  nodes: "nodes.csv"
+  relationships: "relationships.csv"
+  ignore_empty_strings: True
+  import_should_fail: True

tests/integration/mg_import_csv/tests/field_types/expected.cypher

@@ -0,0 +1,4 @@
+CREATE INDEX ON :__mg_vertex__(__mg_id__);
+CREATE (:__mg_vertex__ {__mg_id__: 0, value_bool: false, value_boolean: true, value_integer: 5, value_float: 2.718, value_double: 3.141, value_short: 4, value_str: "hello", id: "0", value_char: "world", value_int: 1, value_long: 2, value_byte: 3});
+DROP INDEX ON :__mg_vertex__(__mg_id__);
+MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;

tests/integration/mg_import_csv/tests/field_types/nodes.csv

@@ -0,0 +1,2 @@
+id:ID,value_str:string,value_char:char,value_int:int,value_long:long,value_byte:byte,value_short:short,value_double:float,value_float:double,value_integer:integer,value_boolean:boolean,value_bool:bool
+0,hello,world,1,2,3,4,3.141,2.718,5,true,foo

tests/integration/mg_import_csv/tests/field_types/test.yaml

@@ -0,0 +1,3 @@
+- name: good_configuration
+  nodes: "nodes.csv"
+  expected: expected.cypher

tests/integration/mg_import_csv/tests/id_type/expected_integer.cypher

@@ -0,0 +1,8 @@
+CREATE INDEX ON :__mg_vertex__(__mg_id__);
+CREATE (:__mg_vertex__:Message:Comment {__mg_id__: 0, content: "yes", browser: "Chrome", country: "Croatia", id: 0});
+CREATE (:__mg_vertex__:Message:Comment {__mg_id__: 1, content: "thanks", browser: "Chrome", country: "United Kingdom", id: 1});
+CREATE (:__mg_vertex__:Message:Comment {__mg_id__: 2, content: "LOL", country: "Germany", id: 2});
+CREATE (:__mg_vertex__:Message:Comment {__mg_id__: 3, content: "I see", browser: "Firefox", country: "France", id: 3});
+CREATE (:__mg_vertex__:Message:Comment {__mg_id__: 4, content: "fine", browser: "Internet Explorer", country: "Italy", id: 4});
+DROP INDEX ON :__mg_vertex__(__mg_id__);
+MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;

tests/integration/mg_import_csv/tests/id_type/expected_string.cypher

@@ -0,0 +1,8 @@
+CREATE INDEX ON :__mg_vertex__(__mg_id__);
+CREATE (:__mg_vertex__:Message:Comment {__mg_id__: 0, content: "yes", browser: "Chrome", country: "Croatia", id: "0"});
+CREATE (:__mg_vertex__:Message:Comment {__mg_id__: 1, content: "thanks", browser: "Chrome", country: "United Kingdom", id: "1"});
+CREATE (:__mg_vertex__:Message:Comment {__mg_id__: 2, content: "LOL", country: "Germany", id: "2"});
+CREATE (:__mg_vertex__:Message:Comment {__mg_id__: 3, content: "I see", browser: "Firefox", country: "France", id: "3"});
+CREATE (:__mg_vertex__:Message:Comment {__mg_id__: 4, content: "fine", browser: "Internet Explorer", country: "Italy", id: "4"});
+DROP INDEX ON :__mg_vertex__(__mg_id__);
+MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;

tests/integration/mg_import_csv/tests/id_type/nodes.csv

@@ -0,0 +1,7 @@
+id:ID(COMMENT_ID),country,browser:string,content:string,:LABEL
+0,Croatia,Chrome,yes,Message;Comment
+1,"United
+ Kingdom",Chrome,thanks,Message;Comment
+2,Germany,,LOL,Message;Comment
+3,France,Firefox,I see,Message;Comment
+4,Italy,Internet Explorer,fine,Message;Comment

tests/integration/mg_import_csv/tests/id_type/test.yaml

@@ -0,0 +1,10 @@
+- name: default_string
+  nodes: "nodes.csv"
+  ignore_empty_strings: True
+  expected: expected_string.cypher
+
+- name: integer
+  nodes: "nodes.csv"
+  ignore_empty_strings: True
+  id_type: "integer"
+  expected: expected_integer.cypher

tests/integration/mg_import_csv/tests/ignore_empty_strings/expected_disabled.cypher

@@ -0,0 +1,8 @@
+CREATE INDEX ON :__mg_vertex__(__mg_id__);
+CREATE (:__mg_vertex__:Message:Comment {__mg_id__: 0, content: "yes", browser: "Chrome", country: "Croatia", id: "0"});
+CREATE (:__mg_vertex__:Message:Comment {__mg_id__: 1, content: "thanks", browser: "Chrome", country: "United Kingdom", id: "1"});
+CREATE (:__mg_vertex__:Message:Comment {__mg_id__: 2, content: "LOL", browser: "", country: "Germany", id: "2"});
+CREATE (:__mg_vertex__:Message:Comment {__mg_id__: 3, content: "I see", browser: "Firefox", country: "France", id: "3"});
+CREATE (:__mg_vertex__:Message:Comment {__mg_id__: 4, content: "fine", browser: "Internet Explorer", country: "Italy", id: "4"});
+DROP INDEX ON :__mg_vertex__(__mg_id__);
+MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;

tests/integration/mg_import_csv/tests/ignore_empty_strings/expected_enabled.cypher

@@ -0,0 +1,8 @@
+CREATE INDEX ON :__mg_vertex__(__mg_id__);
+CREATE (:__mg_vertex__:Message:Comment {__mg_id__: 0, content: "yes", browser: "Chrome", country: "Croatia", id: "0"});
+CREATE (:__mg_vertex__:Message:Comment {__mg_id__: 1, content: "thanks", browser: "Chrome", country: "United Kingdom", id: "1"});
+CREATE (:__mg_vertex__:Message:Comment {__mg_id__: 2, content: "LOL", country: "Germany", id: "2"});
+CREATE (:__mg_vertex__:Message:Comment {__mg_id__: 3, content: "I see", browser: "Firefox", country: "France", id: "3"});
+CREATE (:__mg_vertex__:Message:Comment {__mg_id__: 4, content: "fine", browser: "Internet Explorer", country: "Italy", id: "4"});
+DROP INDEX ON :__mg_vertex__(__mg_id__);
+MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;

tests/integration/mg_import_csv/tests/ignore_empty_strings/nodes.csv

@@ -0,0 +1,7 @@
+id:ID(COMMENT_ID),country,browser:string,content:string,:LABEL
+0,Croatia,Chrome,yes,Message;Comment
+1,"United
+ Kingdom",Chrome,thanks,Message;Comment
+2,Germany,,LOL,Message;Comment
+3,France,Firefox,I see,Message;Comment
+4,Italy,Internet Explorer,fine,Message;Comment

tests/integration/mg_import_csv/tests/ignore_empty_strings/test.yaml

@@ -0,0 +1,8 @@
+- name: default_disabled
+  nodes: "nodes.csv"
+  expected: expected_disabled.cypher
+
+- name: enabled
+  nodes: "nodes.csv"
+  ignore_empty_strings: True
+  expected: expected_enabled.cypher

tests/integration/mg_import_csv/tests/ignored_columns/expected.cypher

@@ -0,0 +1,10 @@
+CREATE INDEX ON :__mg_vertex__(__mg_id__);
+CREATE (:__mg_vertex__:Message:Comment {__mg_id__: 0, country: "Croatia", browser: "Chrome", content: "yes"});
+CREATE (:__mg_vertex__:Message:Comment {__mg_id__: 1, country: "United Kingdom", browser: "Chrome", content: "thanks"});
+CREATE (:__mg_vertex__:Message:Comment {__mg_id__: 2, country: "Germany", content: "LOL"});
+CREATE (:__mg_vertex__:Message:Comment {__mg_id__: 3, country: "France", browser: "Firefox", content: "I see"});
+CREATE (:__mg_vertex__:Message:Comment {__mg_id__: 4, country: "Italy", browser: "Internet Explorer", content: "fine"});
+MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 1 AND v.__mg_id__ = 2 CREATE (u)-[:KNOWS {value: 5}]->(v);
+MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 4 AND v.__mg_id__ = 0 CREATE (u)-[:KNOWS {value: 6}]->(v);
+DROP INDEX ON :__mg_vertex__(__mg_id__);
+MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;

tests/integration/mg_import_csv/tests/ignored_columns/nodes.csv

@@ -0,0 +1,6 @@
+:ID,country:string,browser:string,:IGNORE,content:string,:LABEL,:IGNORE,:IGNORE
+0,Croatia,Chrome,this,yes,Message;Comment,testko,123
+1,"United Kingdom",Chrome,should,thanks,Message;Comment,hello,world
+2,Germany,,not,LOL,Message;Comment,i ignore, everything
+3,France,Firefox,appear,I see,Message;Comment, ignore, ignore
+4,Italy,Internet Explorer,anywhere,fine,Message;Comment, just ingore, it

tests/integration/mg_import_csv/tests/ignored_columns/relationships.csv

@@ -0,0 +1,3 @@
+:START_ID,:IGNORE,:END_ID,:IGNORE,:TYPE,value:int
+1,ignore,2,me,KNOWS,5
+4,hello,0,world,KNOWS,6

tests/integration/mg_import_csv/tests/ignored_columns/test.yaml

@@ -0,0 +1,12 @@
+- name: good_configuration
+  nodes: "nodes.csv"
+  relationships: "relationships.csv"
+  properties_on_edges: True
+  ignore_empty_strings: True
+  expected: expected.cypher
+
+- name: properties_on_edges_disabled
+  nodes: "nodes.csv"
+  relationships: "relationships.csv"
+  properties_on_edges: False
+  import_should_fail: True

tests/integration/mg_import_csv/tests/invalid_double_property_value/nodes.csv

@@ -0,0 +1,2 @@
+:ID,value:double
+0,2.0asd

tests/integration/mg_import_csv/tests/invalid_double_property_value/test.yaml

@@ -0,0 +1,3 @@
+- name: invalid_value
+  nodes: "nodes.csv"
+  import_should_fail: True

tests/integration/mg_import_csv/tests/invalid_integer_node_id_on_node/nodes.csv

@@ -0,0 +1,4 @@
+:ID,country
+1,Austria
+2asd,Hungary
+3,Romania

tests/integration/mg_import_csv/tests/invalid_integer_node_id_on_node/test.yaml

@@ -0,0 +1,4 @@
+- name: invalid_integer_node_id
+  nodes: "nodes.csv"
+  id_type: "integer"
+  import_should_fail: True

tests/integration/mg_import_csv/tests/invalid_integer_node_id_on_relationship/nodes.csv

@@ -0,0 +1,4 @@
+:ID,country
+1,Austria
+2,Hungary
+3,Romania

tests/integration/mg_import_csv/tests/invalid_integer_node_id_on_relationship/relationships.csv

@@ -0,0 +1,2 @@
+:START_ID,:END_ID,:TYPE
+1,2asd,test

tests/integration/mg_import_csv/tests/invalid_integer_node_id_on_relationship/test.yaml

@@ -0,0 +1,5 @@
+- name: invalid_integer_node_id
+  nodes: "nodes.csv"
+  relationships: "relationships.csv"
+  id_type: "integer"
+  import_should_fail: True

tests/integration/mg_import_csv/tests/invalid_integer_property_value/nodes.csv

@@ -0,0 +1,2 @@
+:ID,value:int
+0,2asd

tests/integration/mg_import_csv/tests/invalid_integer_property_value/test.yaml

@@ -0,0 +1,3 @@
+- name: invalid_value
+  nodes: "nodes.csv"
+  import_should_fail: True

tests/integration/mg_import_csv/tests/multi_char_quote_and_node_label/test.yaml

@@ -1,13 +1,11 @@
 - name: good_configuration
   nodes:
-    - nodes_comment.csv
-    - nodes_forum.csv
-  node_label:
-    - First
-    - Second
+    - "First:Second=nodes_comment.csv"
+    - "First:Second=nodes_forum.csv"
   relationships:
     - relationships_0.csv
     - relationships_1.csv
+  ignore_empty_strings: True
   delimiter: "|"
   quote: "Ö"
   array_delimiter: ";"
@@ -15,14 +13,12 @@
 
 - name: wrong_delimiter
   nodes:
-    - nodes_comment.csv
-    - nodes_forum.csv
-  node_label:
-    - First
-    - Second
+    - "First:Second=nodes_comment.csv"
+    - "First:Second=nodes_forum.csv"
   relationships:
     - relationships_0.csv
     - relationships_1.csv
+  ignore_empty_strings: True
   delimiter: ","
   quote: "Ö"
   array_delimiter: ";"
@@ -30,14 +26,12 @@
 
 - name: wrong_quote
   nodes:
-    - nodes_comment.csv
-    - nodes_forum.csv
-  node_label:
-    - First
-    - Second
+    - "First:Second=nodes_comment.csv"
+    - "First:Second=nodes_forum.csv"
   relationships:
     - relationships_0.csv
     - relationships_1.csv
+  ignore_empty_strings: True
   delimiter: "|"
   quote: "\""
   array_delimiter: ";"

tests/integration/mg_import_csv/tests/multiple_same_properties_on_node/nodes.csv

@@ -0,0 +1,2 @@
+:ID,value:int,value:string
+0,2,hello

tests/integration/mg_import_csv/tests/multiple_same_properties_on_node/test.yaml

@@ -0,0 +1,3 @@
+- name: multiple_same_properties
+  nodes: "nodes.csv"
+  import_should_fail: True

tests/integration/mg_import_csv/tests/multiple_same_properties_on_relationship/nodes.csv

@@ -0,0 +1,2 @@
+:ID
+0

tests/integration/mg_import_csv/tests/multiple_same_properties_on_relationship/relationships.csv

@@ -0,0 +1,2 @@
+:START_ID,value,:END_ID,:TYPE,value
+0,5,0,TEST,6

tests/integration/mg_import_csv/tests/multiple_same_properties_on_relationship/test.yaml

@@ -0,0 +1,5 @@
+- name: multiple_same_properties
+  nodes: "nodes.csv"
+  relationships: "relationships.csv"
+  properties_on_edges: True
+  import_should_fail: True