Add new generation option for using proto sources from other frameworks.

- Better docs in the generator for the different options that can be passed
  during an invoke of protoc.
- Add named_framework_to_proto_path_mappings_path to pass the path to a file
  containing mappings of frameworks for different proto files.
- Update the generation to use the mapping to change the #import directives
  it creates.

Note: the changes in helpers is mostly moving code within the fine, and then
a small change to expose the parsing so a passed on class can consume the line.

Fixes https://github.com/google/protobuf/issues/1457

objectivec/README.md

@@ -146,6 +146,29 @@ supported keys are:
     of being plain `#import "some/path/file.pbobjc.h"` lines, they will be
     framework based, i.e. - `#import <VALUE/file.pbobjc.h>`.
 
+    _NOTE:_ If this is used with `named_framework_to_proto_path_mappings_path`,
+    then this is effectively the _default_ to use for everything that wasn't
+    mapped by the other.
+
+  * `named_framework_to_proto_path_mappings_path`: The `value` used for this key
+    is a path to a file containing the listing of framework names and proto
+    files. The generator uses this to decide if another proto file referenced
+    should use a framework style import vs. a user level import
+    (`#import <FRAMEWORK/file.pbobjc.h>` vs `#import "dir/file.pbobjc.h"`).
+
+    The format of the file is:
+      * An entry is a line of `frameworkName: file.proto, dir/file2.proto`.
+      * Comments start with `#`.
+      * A comment can go on a line after an entry.
+        (i.e. - `frameworkName: file.proto # comment`)
+
+    Any number of files can be listed for a framework, just separate them with
+    commas.
+
+    There can be multiple lines listing the same frameworkName incase it has a
+    lot of proto files included in it; and having multiple lines makes things
+    easier to read.
+
 Contributing
 ------------
 

src/google/protobuf/compiler/objectivec/objectivec_file.cc

@@ -37,8 +37,12 @@
 #include <google/protobuf/io/zero_copy_stream_impl.h>
 #include <google/protobuf/stubs/stl_util.h>
 #include <google/protobuf/stubs/strutil.h>
+#include <iostream>
 #include <sstream>
 
+// NOTE: src/google/protobuf/compiler/plugin.cc makes use of cerr for some
+// error cases, so it seems to be ok to use as a back door for errors.
+
 namespace google {
 namespace protobuf {
 
@@ -54,13 +58,31 @@ namespace {
 
 class ImportWriter {
  public:
-  ImportWriter(const Options& options) : options_(options) {}
+  ImportWriter(const Options& options)
+      : options_(options),
+        need_to_parse_mapping_file_(true) {}
 
   void AddFile(const FileGenerator* file);
   void Print(io::Printer *printer) const;
 
  private:
+  class ProtoFrameworkCollector : public LineConsumer {
+   public:
+    ProtoFrameworkCollector(map<string, string>* inout_proto_file_to_framework_name)
+        : map_(inout_proto_file_to_framework_name) {}
+
+    virtual bool ConsumeLine(const StringPiece& line, string* out_error);
+
+   private:
+    map<string, string>* map_;
+  };
+
+  void ParseFrameworkMappings();
+
   const Options options_;
+  map<string, string> proto_file_to_framework_name_;
+  bool need_to_parse_mapping_file_;
+
   vector<string> protobuf_framework_imports_;
   vector<string> protobuf_non_framework_imports_;
   vector<string> other_framework_imports_;
@@ -70,20 +92,39 @@ class ImportWriter {
 void ImportWriter::AddFile(const FileGenerator* file) {
   const FileDescriptor* file_descriptor = file->Descriptor();
   const string extension(".pbobjc.h");
+
   if (IsProtobufLibraryBundledProtoFile(file_descriptor)) {
     protobuf_framework_imports_.push_back(
         FilePathBasename(file_descriptor) + extension);
     protobuf_non_framework_imports_.push_back(file->Path() + extension);
-  } else if (!options_.generate_for_named_framework.empty()) {
+    return;
+  }
+
+  // Lazy parse any mappings.
+  if (need_to_parse_mapping_file_) {
+    ParseFrameworkMappings();
+  }
+
+  map<string, string>::iterator proto_lookup =
+      proto_file_to_framework_name_.find(file_descriptor->name());
+  if (proto_lookup != proto_file_to_framework_name_.end()) {
+    other_framework_imports_.push_back(
+        proto_lookup->second + "/" +
+        FilePathBasename(file_descriptor) + extension);
+    return;
+  }
+
+  if (!options_.generate_for_named_framework.empty()) {
     other_framework_imports_.push_back(
         options_.generate_for_named_framework + "/" +
         FilePathBasename(file_descriptor) + extension);
-  } else {
-    other_imports_.push_back(file->Path() + extension);
+    return;
   }
+
+  other_imports_.push_back(file->Path() + extension);
 }
 
-void ImportWriter::Print(io::Printer *printer) const {
+void ImportWriter::Print(io::Printer* printer) const {
   assert(protobuf_non_framework_imports_.size() ==
          protobuf_framework_imports_.size());
 
@@ -146,6 +187,69 @@ void ImportWriter::Print(io::Printer *printer) const {
   }
 }
 
+void ImportWriter::ParseFrameworkMappings() {
+  need_to_parse_mapping_file_ = false;
+  if (options_.named_framework_to_proto_path_mappings_path.empty()) {
+    return;  // Nothing to do.
+  }
+
+  ProtoFrameworkCollector collector(&proto_file_to_framework_name_);
+  string parse_error;
+  if (!ParseSimpleFile(options_.named_framework_to_proto_path_mappings_path,
+                       &collector, &parse_error)) {
+    cerr << "error parsing " << options_.named_framework_to_proto_path_mappings_path
+         << " : " << parse_error << endl;
+    cerr.flush();
+  }
+}
+
+bool ImportWriter::ProtoFrameworkCollector::ConsumeLine(
+    const StringPiece& line, string* out_error) {
+  int offset = line.find(':');
+  if (offset == StringPiece::npos) {
+    *out_error =
+        string("Framework/proto file mapping line without colon sign: '") +
+        line.ToString() + "'.";
+    return false;
+  }
+  StringPiece framework_name(line, 0, offset);
+  StringPiece proto_file_list(line, offset + 1, line.length() - offset - 1);
+  StringPieceTrimWhitespace(&framework_name);
+
+  int start = 0;
+  while (start < proto_file_list.length()) {
+    offset = proto_file_list.find(',', start);
+    if (offset == StringPiece::npos) {
+      offset = proto_file_list.length();
+    }
+
+    StringPiece proto_file(proto_file_list, start, offset);
+    StringPieceTrimWhitespace(&proto_file);
+    if (proto_file.size() != 0) {
+      map<string, string>::iterator existing_entry =
+          map_->find(proto_file.ToString());
+      if (existing_entry != map_->end()) {
+        cerr << "warning: duplicate proto file reference, replacing framework entry for '"
+             << proto_file.ToString() << "' with '" << framework_name.ToString()
+             << "' (was '" << existing_entry->second << "')." << endl;
+        cerr.flush();
+      }
+
+      if (proto_file.find(' ') != StringPiece::npos) {
+        cerr << "note: framework mapping file had a proto file with a space in, hopefully that isn't a missing comma: '"
+             << proto_file.ToString() << "'" << endl;
+        cerr.flush();
+      }
+
+      (*map_)[proto_file.ToString()] = framework_name.ToString();
+    }
+
+    start = offset + 1;
+  }
+
+  return true;
+}
+
 }  // namespace
 
 

src/google/protobuf/compiler/objectivec/objectivec_generator.cc

@@ -58,9 +58,48 @@ bool ObjectiveCGenerator::Generate(const FileDescriptor* file,
   ParseGeneratorParameter(parameter, &options);
   for (int i = 0; i < options.size(); i++) {
     if (options[i].first == "expected_prefixes_path") {
+      // Path to find a file containing the expected prefixes
+      // (objc_class_prefix "PREFIX") for proto packages (package NAME). The
+      // generator will then issue warnings/errors if in the proto files being
+      // generated the option is not listed/wrong/etc in the file.
+      //
+      // The format of the file is:
+      //   - An entry is a line of "package=prefix".
+      //   - Comments start with "#".
+      //   - A comment can go on a line after a expected package/prefix pair.
+      //     (i.e. - "package=prefix # comment")
+      //
+      // There is no validation that the prefixes are good prefixes, it is
+      // assume they are when you create the file.
       generation_options.expected_prefixes_path = options[i].second;
     } else if (options[i].first == "generate_for_named_framework") {
-      generation_options.generate_for_named_framework = options[i].second;
+      // The name of the framework that protos are being generated for. This
+      // will cause the #import statements to be framework based using this
+      // name (i.e. - "#import <NAME/proto.pbobjc.h>).
+      //
+      // NOTE: If this option is used with
+      // named_framework_to_proto_path_mappings_path, then this is effectively
+      // the "default" to use for everything that wasn't mapped by the other.
+      generation_options.named_framework_to_proto_path_mappings_path = options[i].second;
+    } else if (options[i].first == "named_framework_to_proto_path_mappings_path") {
+      // Path to find a file containing the listing of framework names and
+      // proto files. The generator uses this to decide if another proto file
+      // referenced should use a framework style import vs. a user level import
+      // (#import <FRAMEWORK/file.pbobjc.h> vs #import "dir/file.pbobjc.h").
+      //
+      // The format of the file is:
+      //   - An entry is a line of "frameworkName: file.proto, dir/file2.proto".
+      //   - Comments start with "#".
+      //   - A comment can go on a line after a expected package/prefix pair.
+      //     (i.e. - "frameworkName: file.proto # comment")
+      //
+      // Any number of files can be listed for a framework, just separate them
+      // with commas.
+      //
+      // There can be multiple lines listing the same frameworkName incase it
+      // has a lot of proto files included in it; and having multiple lines
+      // makes things easier to read.
+      generation_options.named_framework_to_proto_path_mappings_path = options[i].second;
     } else {
       *error = "error: Unknown generator option: " + options[i].first;
       return false;

src/google/protobuf/compiler/objectivec/objectivec_helpers.cc

@@ -81,6 +81,10 @@ const char* const kUpperSegmentsList[] = {"url", "http", "https"};
 hash_set<string> kUpperSegments =
     MakeWordsMap(kUpperSegmentsList, GOOGLE_ARRAYSIZE(kUpperSegmentsList));
 
+bool ascii_isnewline(char c) {
+  return c == '\n' || c == '\r';
+}
+
 // Internal helper for name handing.
 // Do not expose this outside of helpers, stick to having functions for specific
 // cases (ClassName(), FieldName()), so there is always consistent suffix rules.
@@ -272,6 +276,16 @@ string StripProto(const string& filename) {
   }
 }
 
+void StringPieceTrimWhitespace(StringPiece* input) {
+  while (!input->empty() && ascii_isspace(*input->data())) {
+    input->remove_prefix(1);
+  }
+  while (!input->empty() && ascii_isspace((*input)[input->length() - 1])) {
+    input->remove_suffix(1);
+  }
+}
+
+
 bool IsRetainedName(const string& name) {
   // List of prefixes from
   // http://developer.apple.com/library/mac/#documentation/Cocoa/Conceptual/MemoryMgmt/Articles/mmRules.html
@@ -871,65 +885,6 @@ bool IsProtobufLibraryBundledProtoFile(const FileDescriptor* file) {
   return false;
 }
 
-namespace {
-
-// Internal helper class that parses the expected package to prefix mappings
-// file.
-class Parser {
- public:
-  Parser(map<string, string>* inout_package_to_prefix_map)
-      : prefix_map_(inout_package_to_prefix_map), line_(0) {}
-
-  // Parses a check of input, returning success/failure.
-  bool ParseChunk(StringPiece chunk);
-
-  // Should be called to finish parsing (after all input has been provided via
-  // ParseChunk()).  Returns success/failure.
-  bool Finish();
-
-  int last_line() const { return line_; }
-  string error_str() const { return error_str_; }
-
- private:
-  bool ParseLoop();
-
-  map<string, string>* prefix_map_;
-  int line_;
-  string error_str_;
-  StringPiece p_;
-  string leftover_;
-};
-
-bool Parser::ParseChunk(StringPiece chunk) {
-  if (!leftover_.empty()) {
-    chunk.AppendToString(&leftover_);
-    p_ = StringPiece(leftover_);
-  } else {
-    p_ = chunk;
-  }
-  bool result = ParseLoop();
-  if (p_.empty()) {
-    leftover_.clear();
-  } else {
-    leftover_ = p_.ToString();
-  }
-  return result;
-}
-
-bool Parser::Finish() {
-  if (leftover_.empty()) {
-    return true;
-  }
-  // Force a newline onto the end to finish parsing.
-  p_ = StringPiece(leftover_ + "\n");
-  if (!ParseLoop()) {
-    return false;
-  }
-  return p_.empty();  // Everything used?
-}
-
-static bool ascii_isnewline(char c) { return c == '\n' || c == '\r'; }
-
 bool ReadLine(StringPiece* input, StringPiece* line) {
   for (int len = 0; len < input->size(); ++len) {
     if (ascii_isnewline((*input)[len])) {
@@ -942,15 +897,6 @@ bool ReadLine(StringPiece* input, StringPiece* line) {
   return false;  // Ran out of input with no newline.
 }
 
-void TrimWhitespace(StringPiece* input) {
-  while (!input->empty() && ascii_isspace(*input->data())) {
-    input->remove_prefix(1);
-  }
-  while (!input->empty() && ascii_isspace((*input)[input->length() - 1])) {
-    input->remove_suffix(1);
-  }
-}
-
 void RemoveComment(StringPiece* input) {
   int offset = input->find('#');
   if (offset != StringPiece::npos) {
@@ -958,29 +904,35 @@ void RemoveComment(StringPiece* input) {
   }
 }
 
-bool Parser::ParseLoop() {
-  StringPiece line;
-  while (ReadLine(&p_, &line)) {
-    ++line_;
-    RemoveComment(&line);
-    TrimWhitespace(&line);
-    if (line.size() == 0) {
-      continue;  // Blank line.
-    }
-    int offset = line.find('=');
-    if (offset == StringPiece::npos) {
-      error_str_ =
-          string("Line without equal sign: '") + line.ToString() + "'.";
-      return false;
-    }
-    StringPiece package(line, 0, offset);
-    StringPiece prefix(line, offset + 1, line.length() - offset - 1);
-    TrimWhitespace(&package);
-    TrimWhitespace(&prefix);
-    // Don't really worry about error checking the package/prefix for
-    // being valid.  Assume the file is validated when it is created/edited.
-    (*prefix_map_)[package.ToString()] = prefix.ToString();
+namespace {
+
+class ExpectedPrefixesCollector : public LineConsumer {
+ public:
+  ExpectedPrefixesCollector(map<string, string>* inout_package_to_prefix_map)
+      : prefix_map_(inout_package_to_prefix_map) {}
+
+  virtual bool ConsumeLine(const StringPiece& line, string* out_error);
+
+ private:
+  map<string, string>* prefix_map_;
+};
+
+bool ExpectedPrefixesCollector::ConsumeLine(
+    const StringPiece& line, string* out_error) {
+  int offset = line.find('=');
+  if (offset == StringPiece::npos) {
+    *out_error =
+        string("Expected prefixes file line without equal sign: '") +
+        line.ToString() + "'.";
+    return false;
   }
+  StringPiece package(line, 0, offset);
+  StringPiece prefix(line, offset + 1, line.length() - offset - 1);
+  StringPieceTrimWhitespace(&package);
+  StringPieceTrimWhitespace(&prefix);
+  // Don't really worry about error checking the package/prefix for
+  // being valid.  Assume the file is validated when it is created/edited.
+  (*prefix_map_)[package.ToString()] = prefix.ToString();
   return true;
 }
 
@@ -991,36 +943,9 @@ bool LoadExpectedPackagePrefixes(const Options &generation_options,
     return true;
   }
 
-  int fd;
-  do {
-    fd = open(generation_options.expected_prefixes_path.c_str(), O_RDONLY);
-  } while (fd < 0 && errno == EINTR);
-  if (fd < 0) {
-    *out_error =
-        string("error: Unable to open \"") +
-        generation_options.expected_prefixes_path +
-        "\", " + strerror(errno);
-    return false;
-  }
-  io::FileInputStream file_stream(fd);
-  file_stream.SetCloseOnDelete(true);
-
-  Parser parser(prefix_map);
-  const void* buf;
-  int buf_len;
-  while (file_stream.Next(&buf, &buf_len)) {
-    if (buf_len == 0) {
-      continue;
-    }
-
-    if (!parser.ParseChunk(StringPiece(static_cast<const char*>(buf), buf_len))) {
-      *out_error =
-          string("error: ") + generation_options.expected_prefixes_path +
-          " Line " + SimpleItoa(parser.last_line()) + ", " + parser.error_str();
-      return false;
-    }
-  }
-  return parser.Finish();
+  ExpectedPrefixesCollector collector(prefix_map);
+  return ParseSimpleFile(
+      generation_options.expected_prefixes_path, &collector, out_error);
 }
 
 }  // namespace
@@ -1121,6 +1046,10 @@ bool ValidateObjCClassPrefix(const FileDescriptor* file,
   return true;
 }
 
+TextFormatDecodeData::TextFormatDecodeData() { }
+
+TextFormatDecodeData::~TextFormatDecodeData() { }
+
 void TextFormatDecodeData::AddString(int32 key,
                                      const string& input_for_decode,
                                      const string& desired_output) {
@@ -1329,6 +1258,116 @@ string TextFormatDecodeData::DecodeDataForString(const string& input_for_decode,
   return builder.Finish() + (char)'\0';
 }
 
+namespace {
+
+class Parser {
+ public:
+  Parser(LineConsumer* line_consumer)
+      : line_consumer_(line_consumer), line_(0) {}
+
+  // Parses a check of input, returning success/failure.
+  bool ParseChunk(StringPiece chunk);
+
+  // Should be called to finish parsing (after all input has been provided via
+  // ParseChunk()).  Returns success/failure.
+  bool Finish();
+
+  int last_line() const { return line_; }
+  string error_str() const { return error_str_; }
+
+ private:
+  bool ParseLoop();
+
+  LineConsumer* line_consumer_;
+  int line_;
+  string error_str_;
+  StringPiece p_;
+  string leftover_;
+};
+
+bool Parser::ParseChunk(StringPiece chunk) {
+  if (!leftover_.empty()) {
+    chunk.AppendToString(&leftover_);
+    p_ = StringPiece(leftover_);
+  } else {
+    p_ = chunk;
+  }
+  bool result = ParseLoop();
+  if (p_.empty()) {
+    leftover_.clear();
+  } else {
+    leftover_ = p_.ToString();
+  }
+  return result;
+}
+
+bool Parser::Finish() {
+  if (leftover_.empty()) {
+    return true;
+  }
+  // Force a newline onto the end to finish parsing.
+  p_ = StringPiece(leftover_ + "\n");
+  if (!ParseLoop()) {
+    return false;
+  }
+  return p_.empty();  // Everything used?
+}
+
+bool Parser::ParseLoop() {
+  StringPiece line;
+  while (ReadLine(&p_, &line)) {
+    ++line_;
+    RemoveComment(&line);
+    StringPieceTrimWhitespace(&line);
+    if (line.size() == 0) {
+      continue;  // Blank line.
+    }
+    if (!line_consumer_->ConsumeLine(line, &error_str_)) {
+      return false;
+    }
+  }
+  return true;
+}
+
+}  // namespace
+
+LineConsumer::LineConsumer() {}
+
+LineConsumer::~LineConsumer() {}
+
+bool ParseSimpleFile(
+    const string& path, LineConsumer* line_consumer, string* out_error) {
+  int fd;
+  do {
+    fd = open(path.c_str(), O_RDONLY);
+  } while (fd < 0 && errno == EINTR);
+  if (fd < 0) {
+    *out_error =
+        string("error: Unable to open \"") + path + "\", " + strerror(errno);
+    return false;
+  }
+  io::FileInputStream file_stream(fd);
+  file_stream.SetCloseOnDelete(true);
+
+  Parser parser(line_consumer);
+  const void* buf;
+  int buf_len;
+  while (file_stream.Next(&buf, &buf_len)) {
+    if (buf_len == 0) {
+      continue;
+    }
+
+    if (!parser.ParseChunk(StringPiece(static_cast<const char*>(buf), buf_len))) {
+      *out_error =
+          string("error: ") + path +
+          " Line " + SimpleItoa(parser.last_line()) + ", " + parser.error_str();
+      return false;
+    }
+  }
+  return parser.Finish();
+}
+
+
 }  // namespace objectivec
 }  // namespace compiler
 }  // namespace protobuf

src/google/protobuf/compiler/objectivec/objectivec_helpers.h

@@ -47,6 +47,7 @@ struct Options {
   Options();
   string expected_prefixes_path;
   string generate_for_named_framework;
+  string named_framework_to_proto_path_mappings_path;
 };
 
 // Escape C++ trigraphs by escaping question marks to "\?".
@@ -55,6 +56,9 @@ string EscapeTrigraphs(const string& to_escape);
 // Strips ".proto" or ".protodevel" from the end of a filename.
 string StripProto(const string& filename);
 
+// Remove white space from either end of a StringPiece.
+void StringPieceTrimWhitespace(StringPiece* input);
+
 // Returns true if the name requires a ns_returns_not_retained attribute applied
 // to it.
 bool IsRetainedName(const string& name);
@@ -190,7 +194,8 @@ bool ValidateObjCClassPrefix(const FileDescriptor* file,
 // the input into the expected output.
 class LIBPROTOC_EXPORT TextFormatDecodeData {
  public:
-  TextFormatDecodeData() {}
+  TextFormatDecodeData();
+  ~TextFormatDecodeData();
 
   void AddString(int32 key, const string& input_for_decode,
                  const string& desired_output);
@@ -207,6 +212,17 @@ class LIBPROTOC_EXPORT TextFormatDecodeData {
   vector<DataEntry> entries_;
 };
 
+// Helper for parsing simple files.
+class LIBPROTOC_EXPORT LineConsumer {
+ public:
+  LineConsumer();
+  virtual ~LineConsumer();
+  virtual bool ConsumeLine(const StringPiece& line, string* out_error) = 0;
+};
+
+bool ParseSimpleFile(
+    const string& path, LineConsumer* line_consumer, string* out_error);
+
 }  // namespace objectivec
 }  // namespace compiler
 }  // namespace protobuf