diff --git a/src/plugins/streamers/oasis/db_plugin/dbOASISWriter.cc b/src/plugins/streamers/oasis/db_plugin/dbOASISWriter.cc index 548a4e447..e50206c45 100644 --- a/src/plugins/streamers/oasis/db_plugin/dbOASISWriter.cc +++ b/src/plugins/streamers/oasis/db_plugin/dbOASISWriter.cc @@ -29,6 +29,14 @@ #include +#include +#include +#include +#include +#include +#include +#include + namespace db { @@ -685,6 +693,10 @@ const size_t cblock_buffer_size = 1024 * 1024; void OASISWriter::write_record_id (char b) { + //if ( m_options.write_step == 211 ) { + // std::cout << "[DEBUG] step 21 write rid " << static_cast(b) << std::endl ; + //} + if (m_in_cblock) { if (m_cblock_buffer.size () > cblock_buffer_size) { end_cblock (); @@ -1497,6 +1509,8 @@ OASISWriter::write (db::Layout &layout, tl::OutputStream &stream, const db::Save tl::warn << msg; } + //std::cout << "[DEBUG] pos1 " << mp_stream->pos () << std::endl ; + double dbu = (options.dbu () == 0.0) ? layout.dbu () : options.dbu (); m_sf = options.scale_factor () * (layout.dbu () / dbu); if (fabs (m_sf - 1.0) < 1e-9) { @@ -1528,17 +1542,7 @@ OASISWriter::write (db::Layout &layout, tl::OutputStream &stream, const db::Save } } - // write header - - char magic[] = "%SEMI-OASIS\015\012"; - write_bytes (magic, sizeof (magic) - 1); - - // START record - write_record_id (1); - write_bstring ("1.0"); - write (1.0 / dbu); - write_byte (m_options.strict_mode ? 1 : 0); // offset-flag (strict mode: at the end, non-strict mode: at the beginning) - + //std::cout << "[DEBUG] pos2 " << mp_stream->pos () << std::endl ; size_t cellnames_table_pos = 0; size_t textstrings_table_pos = 0; size_t propnames_table_pos = 0; @@ -1546,15 +1550,110 @@ OASISWriter::write (db::Layout &layout, tl::OutputStream &stream, const db::Save size_t layernames_table_pos = 0; std::map cell_positions; - if (! m_options.strict_mode) { + //std::shared_ptr cell_bytes_stream; + //std::shared_ptr cell_bytes_stream; - // offset table: - for (unsigned int i = 0; i < 12; ++i) { - write_byte (0); + constexpr off_t cache_mmap_threshold = 1 * 1024 * 1024; // 1M + + std::vector cell_bytes_buffer; + const char* bytes_cache_data = nullptr; + bool use_mmap_cache = false; + off_t bytes_cache_file_size = 0; + + if ( m_options.write_step == 4 || m_options.write_step == 41 ) { + long stream_data_size = m_options.get_stream_data_size(); + + if ( stream_data_size > -1 ) { + const uint8_t* data_in = m_options.get_stream_data(); + bytes_cache_data = reinterpret_cast(data_in); + + bytes_cache_file_size = stream_data_size; + + } else { + // load file + const char* bytes_cache_fname = m_options.get_stream_str(); + + int fd = open(bytes_cache_fname, O_RDONLY); + if (fd == -1) { + std::string msg = tl::to_string (tr ("fail to open cache file: ")) + bytes_cache_fname; + throw tl::Exception (msg); + } + + struct stat sb; + if (fstat(fd, &sb) == -1) { + std::string msg = tl::to_string (tr ("fail to get stat of cache file: ")) + bytes_cache_fname; + close(fd); + throw tl::Exception (msg); + } + + bytes_cache_file_size = sb.st_size; + + //std::cout << "[DEBUG] cache " << bytes_cache_fname << " , size " << bytes_cache_file_size << std::endl ; + + if ( bytes_cache_file_size == 0 ) { + // empty file + bytes_cache_data = nullptr; + + } else if ( bytes_cache_file_size < cache_mmap_threshold ) { + // small file, read once + cell_bytes_buffer.resize(bytes_cache_file_size); + ssize_t n = read(fd, cell_bytes_buffer.data(), bytes_cache_file_size); + if ( n != bytes_cache_file_size ) { + std::string msg = tl::to_string (tr ("fail to read cache file: ")) + bytes_cache_fname; + close(fd); + throw tl::Exception (msg); + } + bytes_cache_data = cell_bytes_buffer.data(); + close(fd); + + } else { + // large file, use mmap + void* mmap_ptr = mmap( + nullptr, + bytes_cache_file_size, + PROT_READ, + MAP_PRIVATE, + fd, + 0 // map from head + ); + + if (mmap_ptr == MAP_FAILED) { + std::string msg = tl::to_string (tr ("fail to mmap cache file: ")) + bytes_cache_fname; + close(fd); + throw tl::Exception (msg); + } + + bytes_cache_data = static_cast(mmap_ptr); + use_mmap_cache = true; + close(fd); + } } + } + + if ( m_options.write_step == 2 || m_options.write_step == 21 || m_options.write_step == 3) { + // write_step 2, only cell, + // write_step 3, only end, + } else { + char magic[] = "%SEMI-OASIS\015\012"; + write_bytes (magic, sizeof (magic) - 1); + + // START record + write_record_id (1); + write_bstring ("1.0"); + write (1.0 / dbu); + write_byte (m_options.strict_mode ? 1 : 0); // offset-flag (strict mode: at the end, non-strict mode: at the beginning) + + if (! m_options.strict_mode) { + + // offset table: + for (unsigned int i = 0; i < 12; ++i) { + write_byte (0); + } + } } + //std::cout << "[DEBUG] pos3 " << mp_stream->pos () << std::endl ; // Reset the global variables reset_modal_variables (); @@ -1589,37 +1688,48 @@ OASISWriter::write (db::Layout &layout, tl::OutputStream &stream, const db::Save } } } + //std::cout << "[DEBUG] pos4 " << mp_stream->pos () << std::endl ; - if (m_options.write_std_properties > 0) { - - write_property_def (s_max_signed_integer_width_name, tl::Variant (sizeof (db::Coord)), true); - write_property_def (s_max_unsigned_integer_width_name, tl::Variant (sizeof (coord_distance_type)), true); - - for (std::vector::const_iterator cell = cells.begin (); cell != cells.end (); ++cell) { - const db::Cell &c = layout.cell (*cell); - bool is_top = true; - for (db::Cell::parent_cell_iterator p = c.begin_parent_cells (); p != c.end_parent_cells () && is_top; ++p) { - is_top = (cell_set.find (*p) == cell_set.end ()); + if ( m_options.write_step == 2 || m_options.write_step == 21 || m_options.write_step == 3) { + // write_step 2, only cell, + // write_step 3, only end, + } else { + if (m_options.write_std_properties > 0) { + + write_property_def (s_max_signed_integer_width_name, tl::Variant (sizeof (db::Coord)), true); + write_property_def (s_max_unsigned_integer_width_name, tl::Variant (sizeof (coord_distance_type)), true); + + for (std::vector::const_iterator cell = cells.begin (); cell != cells.end (); ++cell) { + const db::Cell &c = layout.cell (*cell); + bool is_top = true; + for (db::Cell::parent_cell_iterator p = c.begin_parent_cells (); p != c.end_parent_cells () && is_top; ++p) { + is_top = (cell_set.find (*p) == cell_set.end ()); + } + if (is_top) { + write_property_def (s_top_cell_name, tl::Variant (make_nstring (layout.cell_name (*cell))), true); + } } - if (is_top) { - write_property_def (s_top_cell_name, tl::Variant (make_nstring (layout.cell_name (*cell))), true); + + if (m_options.write_std_properties > 1) { + write_property_def (s_bounding_boxes_available_name, tl::Variant ((unsigned int) 2), true); } } - - if (m_options.write_std_properties > 1) { - write_property_def (s_bounding_boxes_available_name, tl::Variant ((unsigned int) 2), true); - } - } if (m_options.write_std_properties > 1) { m_propnames.insert (std::make_pair (s_bounding_box_name, m_propname_id++)); } - if (layout.prop_id () != 0) { - write_props (layout.prop_id ()); + if ( m_options.write_step == 2 || m_options.write_step == 21 || m_options.write_step == 3) { + // write_step 2, only cell, + // write_step 3, only end, + } else { + if (layout.prop_id () != 0) { + write_props (layout.prop_id ()); + } } + //std::cout << "[DEBUG] pos5 " << mp_stream->pos () << std::endl ; std::vector context_prop_strings; // write the global layout context information @@ -1632,35 +1742,58 @@ OASISWriter::write (db::Layout &layout, tl::OutputStream &stream, const db::Save values.push_back (tl::Variant (*i)); } - write_property_def (klayout_context_name, values, false); + if ( m_options.write_step == 2 || m_options.write_step == 21 || m_options.write_step == 3) { + // write_step 2, only cell, + // write_step 3, only end, + } else { + write_property_def (klayout_context_name, values, false); + } context_prop_strings.clear (); } - // write the tables - + //std::cout << "[DEBUG] pos6 " << mp_stream->pos () << std::endl ; if (! m_options.tables_at_end) { - write_propname_table (propnames_table_pos, cells, layout, layers); - write_propstring_table (propstrings_table_pos, cells, layout, layers); + if ( m_options.write_step == 2 || m_options.write_step == 21 || m_options.write_step == 3) { + // write_step 2, only cell, + // write_step 3, only end, + } else { + write_propname_table (propnames_table_pos, cells, layout, layers); + write_propstring_table (propstrings_table_pos, cells, layout, layers); + } // Now we cannot open new property ID's in strict mode m_proptables_written = true; - // build cell name table now in non-strict mode (in strict mode it is written at the - // end because then we have the cell positions fo S_CELL_OFFSET) - if (! m_options.strict_mode) { - write_cellname_table (cellnames_table_pos, cells_by_index, 0, layout); - } + if ( m_options.write_step == 2 || m_options.write_step == 21 || m_options.write_step == 3) { + // write_step 2, only cell, + // write_step 3, only end, + } else { + // build cell name table now in non-strict mode (in strict mode it is written at the + // end because then we have the cell positions fo S_CELL_OFFSET) + if (! m_options.strict_mode) { + write_cellname_table (cellnames_table_pos, cells_by_index, 0, layout); + } - write_textstring_table (textstrings_table_pos, cells, layout, layers); - write_layername_table (layernames_table_pos, layers); + write_textstring_table (textstrings_table_pos, cells, layout, layers); + write_layername_table (layernames_table_pos, layers); + } + } + //std::cout << "[DEBUG] pos7 " << mp_stream->pos () << std::endl ; + if ( m_options.write_step == 1) { + // write_step 1, only head + return; } // write cells + if ( m_options.write_step == 2 || m_options.write_step == 21 ) { + layout.clear_bytes_cache(); + } + for (std::vector::const_iterator cell = cells.begin (); cell != cells.end (); ++cell) { m_progress.set (mp_stream->pos ()); @@ -1668,16 +1801,85 @@ OASISWriter::write (db::Layout &layout, tl::OutputStream &stream, const db::Save // cell body const db::Cell &cref (layout.cell (*cell)); mp_cell = &cref; + //std::cout << "[DEBUG] cell " << mp_cell->cell_index () << " pos " << mp_stream->pos () << std::endl ; // skip cell body if the cell is not to be written if (skip_cell_body (cref)) { continue; } + if ( m_options.write_step == 3) { + continue; + } // cell header + size_t pos_start = mp_stream->pos () ; + + if ( m_options.write_step == 2 || m_options.write_step == 21 ) { + //if ( m_options.write_one_cell != *cell ) { + // continue; + //} + + if ( m_options.write_one_cell.find(*cell) != m_options.write_one_cell.end()) { + // find cid to write + } else { + // not in write list, skip + continue; + } + } + + if ( m_options.write_step == 21 ) { + //m_options.write_step = 211; + reset_modal_variables (); + + // shapes + for (std::vector >::const_iterator l = layers.begin (); l != layers.end (); ++l) { + const db::Shapes &shapes = cref.shapes (l->first); + if (! shapes.empty ()) { + write_shapes (l->second, shapes); + //m_progress.set (mp_stream->pos ()); + } + } + + size_t data_len = mp_stream->pos () - pos_start; + layout.set_bytes_cache(*cell, pos_start, data_len); + + //m_options.write_step = 21; + continue; + } + + //std::cout << "[DEBUG] cell " << *cell << " pos start " << mp_stream->pos () << std::endl ; cell_positions.insert (std::make_pair (*cell, mp_stream->pos ())); + if ( m_options.write_step == 4 ) { + + auto pos_len = layout.get_bytes_cache(*cell); + if (pos_len) { + //std::cout << "[DEBUG] found " << pos_len->first << "," << pos_len->second << std::endl ; + + //cell_bytes_stream->seek(pos_len->first); + + //char data[pos_len->second]; + //cell_bytes_stream->read(data, sizeof(data)); + + //mp_stream->put(data, pos_len->second); + off_t offset = pos_len->first; + size_t length = pos_len->second; + + //std::cout << "[DEBUG] size " << offset << "," << length << "," << bytes_cache_file_size << std::endl ; + if ( offset + length <= bytes_cache_file_size ) { + const char* data = bytes_cache_data + offset; + mp_stream->put(data, length); + + } else { + std::string msg = tl::to_string (tr ("invail data buffer range")); + throw tl::Exception (msg); + } + + continue; + } + } + write_record_id (13); // CELL write ((unsigned long) *cell); @@ -1738,12 +1940,36 @@ OASISWriter::write (db::Layout &layout, tl::OutputStream &stream, const db::Save write_insts (cell_set); } - // shapes - for (std::vector >::const_iterator l = layers.begin (); l != layers.end (); ++l) { - const db::Shapes &shapes = cref.shapes (l->first); - if (! shapes.empty ()) { - write_shapes (l->second, shapes); - m_progress.set (mp_stream->pos ()); + int write_shape_bytes = 0 ; + + if ( m_options.write_step == 41 ) { + auto pos_len = layout.get_bytes_cache(*cell); + if (pos_len) { + write_shape_bytes = 1; + + off_t offset = pos_len->first; + size_t length = pos_len->second; + + //std::cout << "[DEBUG] size " << offset << "," << length << "," << bytes_cache_file_size << std::endl ; + if ( offset + length <= bytes_cache_file_size ) { + const char* data = bytes_cache_data + offset; + //mp_stream->put(data, length); + write_bytes (data, length); + + } else { + std::string msg = tl::to_string (tr ("invail data buffer range")); + throw tl::Exception (msg); + } + } + } + + if ( write_shape_bytes == 0) { + for (std::vector >::const_iterator l = layers.begin (); l != layers.end (); ++l) { + const db::Shapes &shapes = cref.shapes (l->first); + if (! shapes.empty ()) { + write_shapes (l->second, shapes); + m_progress.set (mp_stream->pos ()); + } } } @@ -1753,10 +1979,33 @@ OASISWriter::write (db::Layout &layout, tl::OutputStream &stream, const db::Save } // end of cell + //std::cout << "[DEBUG] cell " << *cell << " pos end " << mp_stream->pos () << std::endl ; + + if ( m_options.write_step == 2 ) { + size_t data_len = mp_stream->pos () - pos_start; + layout.set_bytes_cache(*cell, pos_start, data_len); + } + + } + if ( m_options.write_step == 2 || m_options.write_step == 21 ) { + // write_step 2, only cell + return ; } +step_at_end: // write the tables if at end + //std::cout << "[DEBUG] end p1 " << mp_stream->pos () << std::endl ; + //std::cout << "[DEBUG] tables at end " << m_options.tables_at_end << std::endl ; + + if ( m_options.write_step == 4 || m_options.write_step == 41 ) { + if (use_mmap_cache) { + if (munmap(const_cast(bytes_cache_data), bytes_cache_file_size) == -1) { + std::string msg = tl::to_string (tr ("failed to munmap")); + throw tl::Exception (msg); + } + } + } if (m_options.tables_at_end) { @@ -2115,6 +2364,11 @@ OASISWriter::write_insts (const std::set &cell_set) void OASISWriter::write_props (db::properties_id_type prop_id) { + if ( m_options.write_step == 2 || m_options.write_step == 21 ) { + //TODO , need to improve, skip prop now + return ; + } + std::vector pv_list; auto props = db::properties (prop_id).to_map (); @@ -2195,6 +2449,12 @@ OASISWriter::write_property_def (const char *name_str, const std::vector -1 ) { + // std::cout << "[DEBUG] write_step " << m_options.write_step + // << " m_proptables_written " << m_proptables_written << std::endl ; + //} + if (pni == m_propnames.end () && m_options.strict_mode) { tl_assert (! m_proptables_written); pni = m_propnames.insert (std::make_pair (name_str, m_propname_id++)).first;