Changeset 7d22e19


Ignore:
Timestamp:
Sep 1, 2017 2:20:38 PM (3 weeks ago)
Author:
www-data <www-data@…>
Branches:
master
Children:
e4b76c9
Parents:
eefd714
git-author:
Bang Pham Huu <b.phamhuu@…> (08/31/17 13:49:10)
git-committer:
www-data <www-data@…> (09/01/17 14:20:38)
Message:

ticket:971 - WCST_Import reports time, speed to ingest each slice

Summary:
WCST_Import needs to report time, speed (MB/s) to ingest each slice (file).
Example:
Progress: [------------------------------] 0/4 0.00%
Test me: AUT-PM10_2008-01-01T02:01:20.geo.tif with size: 0.02 MB. Total time to ingest: 0.71 s. Speed: 0.03 MB/s.

Test Plan: Run test wcst_import

Reviewers: dmisev

Differential Revision: http://codereview.rasdaman.org/D462

Files:
3 edited

Legend:

Unmodified
Added
Removed
  • applications/petascope/petascope_core/src/main/java/petascope/util/ras/TypeRegistry.java

    rb6a2369 r7d22e19  
    125125        StringBuilder result = new StringBuilder();
    126126        for (int i = 0 ; i < numberOfDimensions; i++) {
    127             result.append("a" + String.valueOf(i));
     127            result.append("D" + String.valueOf(i));
    128128            if (i < numberOfDimensions - 1) {
    129129                result.append(",");
  • applications/wcst_import/master/importer/importer.py

    rd77499d r7d22e19  
    2424
    2525import decimal
     26import os
     27import time
    2628from lib import arrow
    2729from collections import OrderedDict
     
    136138        for i in range(self.processed, self.total):
    137139            try:
    138                 self._insert_slice(self.coverage.slices[i])
     140                # Log the time to send the slice (file) to server to ingest
     141                # NOTE: in case of using wcs_extract recipe, it will fetch file from server, so don't know the file size
     142                if hasattr(self.coverage.slices[i].data_provider, "file"):
     143                    file_path = self.coverage.slices[i].data_provider.file.filepath
     144                    file_size_in_mb = round((float)(os.path.getsize(file_path)) / (1000*1000), 2)
     145                    file_name = os.path.basename(file_path)
     146                    start_time = time.time()
     147                    self._insert_slice(self.coverage.slices[i])
     148                    end_time = time.time()
     149                    time_to_ingest = round(end_time - start_time, 2)
     150                    size_per_second = round(file_size_in_mb / time_to_ingest, 2)
     151                    log.info("\nFile name: " + file_name + " with size: " + str(file_size_in_mb) + " MB. " +
     152                             "Total time to ingest: " + str(time_to_ingest) + " s. Speed: " + str(size_per_second) + " MB/s.")
     153                else:
     154                    # extract coverage from petascope to ingest a new coverage
     155                    start_time = time.time()
     156                    self._insert_slice(self.coverage.slices[i])
     157                    end_time = time.time()
     158                    time_to_ingest = round(end_time - start_time, 2)
     159                    log.info("\nTotal time to ingest: " + str(time_to_ingest) + " s.")
     160
    139161            except Exception as e:
    140162                if ConfigManager.skip:
  • systemtest/testcases_services/test_wms/known_fails

    r1b40e87 r7d22e19  
     122_project_3_layers_styles_4326_to_3857.test
Note: See TracChangeset for help on using the changeset viewer.