gdal2tiles_parallelize_base_and_overview_tiles.patch 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345
  1. --- /usr/bin/gdal2tiles.py 2011-11-24 16:20:02.340068830 +0100
  2. +++ gdal2tiles.py 2011-12-12 02:35:30.008308688 +0100
  3. @@ -56,11 +56,16 @@
  4. # 'antialias' resampling is not available
  5. pass
  6. +import multiprocessing
  7. +import tempfile
  8. +from optparse import OptionParser, OptionGroup
  9. +
  10. __version__ = "$Id: gdal2tiles.py 19288 2010-04-02 18:36:17Z rouault $"
  11. resampling_list = ('average','near','bilinear','cubic','cubicspline','lanczos','antialias')
  12. profile_list = ('mercator','geodetic','raster') #,'zoomify')
  13. webviewer_list = ('all','google','openlayers','none')
  14. +queue = multiprocessing.Queue()
  15. # =============================================================================
  16. # =============================================================================
  17. @@ -98,8 +103,6 @@
  18. Class is available under the open-source GDAL license (www.gdal.org).
  19. """
  20. -import math
  21. -
  22. MAXZOOMLEVEL = 32
  23. class GlobalMercator(object):
  24. @@ -640,7 +643,6 @@
  25. def optparse_init(self):
  26. """Prepare the option parser for input (argv)"""
  27. - from optparse import OptionParser, OptionGroup
  28. usage = "Usage: %prog [options] input_file(s) [output]"
  29. p = OptionParser(usage, version="%prog "+ __version__)
  30. p.add_option("-p", "--profile", dest='profile', type='choice', choices=profile_list,
  31. @@ -655,6 +657,8 @@
  32. help="Resume mode. Generate only missing files.")
  33. p.add_option('-a', '--srcnodata', dest="srcnodata", metavar="NODATA",
  34. help="NODATA transparency value to assign to the input data")
  35. + p.add_option('--processes', dest='processes', type='int', default=multiprocessing.cpu_count(),
  36. + help='Number of concurrent processes (defaults to the number of cores in the system)')
  37. p.add_option("-v", "--verbose",
  38. action="store_true", dest="verbose",
  39. help="Print status messages to stdout")
  40. @@ -699,7 +703,10 @@
  41. def open_input(self):
  42. """Initialization of the input raster, reprojection if necessary"""
  43. + gdal.UseExceptions()
  44. gdal.AllRegister()
  45. + if not self.options.verbose:
  46. + gdal.PushErrorHandler('CPLQuietErrorHandler')
  47. # Initialize necessary GDAL drivers
  48. @@ -816,8 +823,7 @@
  49. # Correction of AutoCreateWarpedVRT for NODATA values
  50. if self.in_nodata != []:
  51. - import tempfile
  52. - tempfilename = tempfile.mktemp('-gdal2tiles.vrt')
  53. + fd, tempfilename = tempfile.mkstemp('-gdal2tiles.vrt')
  54. self.out_ds.GetDriver().CreateCopy(tempfilename, self.out_ds)
  55. # open as a text file
  56. s = open(tempfilename).read()
  57. @@ -851,8 +857,7 @@
  58. # Correction of AutoCreateWarpedVRT for Mono (1 band) and RGB (3 bands) files without NODATA:
  59. # equivalent of gdalwarp -dstalpha
  60. if self.in_nodata == [] and self.out_ds.RasterCount in [1,3]:
  61. - import tempfile
  62. - tempfilename = tempfile.mktemp('-gdal2tiles.vrt')
  63. + fd, tempfilename = tempfile.mkstemp('-gdal2tiles.vrt')
  64. self.out_ds.GetDriver().CreateCopy(tempfilename, self.out_ds)
  65. # open as a text file
  66. s = open(tempfilename).read()
  67. @@ -1139,11 +1144,9 @@
  68. f.close()
  69. # -------------------------------------------------------------------------
  70. - def generate_base_tiles(self):
  71. + def generate_base_tiles(self, cpu):
  72. """Generation of the base tiles (the lowest in the pyramid) directly from the input raster"""
  73. - print("Generating Base Tiles:")
  74. -
  75. if self.options.verbose:
  76. #mx, my = self.out_gt[0], self.out_gt[3] # OriginX, OriginY
  77. #px, py = self.mercator.MetersToPixels( mx, my, self.tmaxz)
  78. @@ -1183,6 +1186,8 @@
  79. if self.stopped:
  80. break
  81. ti += 1
  82. + if (ti - 1) % self.options.processes != cpu:
  83. + continue
  84. tilefilename = os.path.join(self.output, str(tz), str(tx), "%s.%s" % (ty, self.tileext))
  85. if self.options.verbose:
  86. print(ti,'/',tcount, tilefilename) #, "( TileMapService: z / x / y )"
  87. @@ -1191,7 +1196,7 @@
  88. if self.options.verbose:
  89. print("Tile generation skiped because of --resume")
  90. else:
  91. - self.progressbar( ti / float(tcount) )
  92. + queue.put(tcount)
  93. continue
  94. # Create directories for the tile
  95. @@ -1296,101 +1301,100 @@
  96. f.close()
  97. if not self.options.verbose:
  98. - self.progressbar( ti / float(tcount) )
  99. + queue.put(tcount)
  100. # -------------------------------------------------------------------------
  101. - def generate_overview_tiles(self):
  102. + def generate_overview_tiles(self, cpu, tz):
  103. """Generation of the overview tiles (higher in the pyramid) based on existing tiles"""
  104. - print("Generating Overview Tiles:")
  105. -
  106. tilebands = self.dataBandsCount + 1
  107. # Usage of existing tiles: from 4 underlying tiles generate one as overview.
  108. tcount = 0
  109. - for tz in range(self.tmaxz-1, self.tminz-1, -1):
  110. - tminx, tminy, tmaxx, tmaxy = self.tminmax[tz]
  111. + for z in range(self.tmaxz-1, self.tminz-1, -1):
  112. + tminx, tminy, tmaxx, tmaxy = self.tminmax[z]
  113. tcount += (1+abs(tmaxx-tminx)) * (1+abs(tmaxy-tminy))
  114. ti = 0
  115. # querysize = tilesize * 2
  116. - for tz in range(self.tmaxz-1, self.tminz-1, -1):
  117. - tminx, tminy, tmaxx, tmaxy = self.tminmax[tz]
  118. - for ty in range(tmaxy, tminy-1, -1): #range(tminy, tmaxy+1):
  119. - for tx in range(tminx, tmaxx+1):
  120. + tminx, tminy, tmaxx, tmaxy = self.tminmax[tz]
  121. + for ty in range(tmaxy, tminy-1, -1): #range(tminy, tmaxy+1):
  122. + for tx in range(tminx, tmaxx+1):
  123. +
  124. + if self.stopped:
  125. + break
  126. - if self.stopped:
  127. - break
  128. -
  129. - ti += 1
  130. - tilefilename = os.path.join( self.output, str(tz), str(tx), "%s.%s" % (ty, self.tileext) )
  131. + ti += 1
  132. + if (ti - 1) % self.options.processes != cpu:
  133. + continue
  134. + tilefilename = os.path.join( self.output, str(tz), str(tx), "%s.%s" % (ty, self.tileext) )
  135. + if self.options.verbose:
  136. + print(ti,'/',tcount, tilefilename) #, "( TileMapService: z / x / y )"
  137. +
  138. + if self.options.resume and os.path.exists(tilefilename):
  139. if self.options.verbose:
  140. - print(ti,'/',tcount, tilefilename) #, "( TileMapService: z / x / y )"
  141. -
  142. - if self.options.resume and os.path.exists(tilefilename):
  143. - if self.options.verbose:
  144. - print("Tile generation skiped because of --resume")
  145. - else:
  146. - self.progressbar( ti / float(tcount) )
  147. - continue
  148. -
  149. - # Create directories for the tile
  150. - if not os.path.exists(os.path.dirname(tilefilename)):
  151. - os.makedirs(os.path.dirname(tilefilename))
  152. + print("Tile generation skiped because of --resume")
  153. + else:
  154. + queue.put(tcount)
  155. + continue
  156. - dsquery = self.mem_drv.Create('', 2*self.tilesize, 2*self.tilesize, tilebands)
  157. - # TODO: fill the null value
  158. - #for i in range(1, tilebands+1):
  159. - # dsquery.GetRasterBand(1).Fill(tilenodata)
  160. - dstile = self.mem_drv.Create('', self.tilesize, self.tilesize, tilebands)
  161. + # Create directories for the tile
  162. + if not os.path.exists(os.path.dirname(tilefilename)):
  163. + os.makedirs(os.path.dirname(tilefilename))
  164. - # TODO: Implement more clever walking on the tiles with cache functionality
  165. - # probably walk should start with reading of four tiles from top left corner
  166. - # Hilbert curve...
  167. -
  168. - children = []
  169. - # Read the tiles and write them to query window
  170. - for y in range(2*ty,2*ty+2):
  171. - for x in range(2*tx,2*tx+2):
  172. - minx, miny, maxx, maxy = self.tminmax[tz+1]
  173. - if x >= minx and x <= maxx and y >= miny and y <= maxy:
  174. - dsquerytile = gdal.Open( os.path.join( self.output, str(tz+1), str(x), "%s.%s" % (y, self.tileext)), gdal.GA_ReadOnly)
  175. - if (ty==0 and y==1) or (ty!=0 and (y % (2*ty)) != 0):
  176. - tileposy = 0
  177. - else:
  178. - tileposy = self.tilesize
  179. - if tx:
  180. - tileposx = x % (2*tx) * self.tilesize
  181. - elif tx==0 and x==1:
  182. - tileposx = self.tilesize
  183. - else:
  184. - tileposx = 0
  185. - dsquery.WriteRaster( tileposx, tileposy, self.tilesize, self.tilesize,
  186. - dsquerytile.ReadRaster(0,0,self.tilesize,self.tilesize),
  187. - band_list=list(range(1,tilebands+1)))
  188. - children.append( [x, y, tz+1] )
  189. + dsquery = self.mem_drv.Create('', 2*self.tilesize, 2*self.tilesize, tilebands)
  190. + # TODO: fill the null value
  191. + #for i in range(1, tilebands+1):
  192. + # dsquery.GetRasterBand(1).Fill(tilenodata)
  193. + dstile = self.mem_drv.Create('', self.tilesize, self.tilesize, tilebands)
  194. - self.scale_query_to_tile(dsquery, dstile, tilefilename)
  195. + # TODO: Implement more clever walking on the tiles with cache functionality
  196. + # probably walk should start with reading of four tiles from top left corner
  197. + # Hilbert curve...
  198. +
  199. + children = []
  200. + # Read the tiles and write them to query window
  201. + for y in range(2*ty,2*ty+2):
  202. + for x in range(2*tx,2*tx+2):
  203. + minx, miny, maxx, maxy = self.tminmax[tz+1]
  204. + if x >= minx and x <= maxx and y >= miny and y <= maxy:
  205. + dsquerytile = gdal.Open( os.path.join( self.output, str(tz+1), str(x), "%s.%s" % (y, self.tileext)), gdal.GA_ReadOnly)
  206. + if (ty==0 and y==1) or (ty!=0 and (y % (2*ty)) != 0):
  207. + tileposy = 0
  208. + else:
  209. + tileposy = self.tilesize
  210. + if tx:
  211. + tileposx = x % (2*tx) * self.tilesize
  212. + elif tx==0 and x==1:
  213. + tileposx = self.tilesize
  214. + else:
  215. + tileposx = 0
  216. + dsquery.WriteRaster( tileposx, tileposy, self.tilesize, self.tilesize,
  217. + dsquerytile.ReadRaster(0,0,self.tilesize,self.tilesize),
  218. + band_list=list(range(1,tilebands+1)))
  219. + children.append( [x, y, tz+1] )
  220. +
  221. + self.scale_query_to_tile(dsquery, dstile, tilefilename)
  222. + # Write a copy of tile to png/jpg
  223. + if self.options.resampling != 'antialias':
  224. # Write a copy of tile to png/jpg
  225. - if self.options.resampling != 'antialias':
  226. - # Write a copy of tile to png/jpg
  227. - self.out_drv.CreateCopy(tilefilename, dstile, strict=0)
  228. + self.out_drv.CreateCopy(tilefilename, dstile, strict=0)
  229. - if self.options.verbose:
  230. - print("\tbuild from zoom", tz+1," tiles:", (2*tx, 2*ty), (2*tx+1, 2*ty),(2*tx, 2*ty+1), (2*tx+1, 2*ty+1))
  231. + if self.options.verbose:
  232. + print("\tbuild from zoom", tz+1," tiles:", (2*tx, 2*ty), (2*tx+1, 2*ty),(2*tx, 2*ty+1), (2*tx+1, 2*ty+1))
  233. - # Create a KML file for this tile.
  234. - if self.kml:
  235. - f = open( os.path.join(self.output, '%d/%d/%d.kml' % (tz, tx, ty)), 'w')
  236. - f.write( self.generate_kml( tx, ty, tz, children ) )
  237. - f.close()
  238. + # Create a KML file for this tile.
  239. + if self.kml:
  240. + f = open( os.path.join(self.output, '%d/%d/%d.kml' % (tz, tx, ty)), 'w')
  241. + f.write( self.generate_kml( tx, ty, tz, children ) )
  242. + f.close()
  243. - if not self.options.verbose:
  244. - self.progressbar( ti / float(tcount) )
  245. + if not self.options.verbose:
  246. + queue.put(tcount)
  247. # -------------------------------------------------------------------------
  248. @@ -2234,8 +2238,64 @@
  249. # =============================================================================
  250. # =============================================================================
  251. +def worker_metadata(argv):
  252. + gdal2tiles = GDAL2Tiles( argv[1:] )
  253. + gdal2tiles.open_input()
  254. + gdal2tiles.generate_metadata()
  255. +
  256. +def worker_base_tiles(argv, cpu):
  257. + gdal2tiles = GDAL2Tiles( argv[1:] )
  258. + gdal2tiles.open_input()
  259. + gdal2tiles.generate_base_tiles(cpu)
  260. +
  261. +def worker_overview_tiles(argv, cpu, tz):
  262. + gdal2tiles = GDAL2Tiles( argv[1:] )
  263. + gdal2tiles.open_input()
  264. + gdal2tiles.generate_overview_tiles(cpu, tz)
  265. +
  266. if __name__=='__main__':
  267. argv = gdal.GeneralCmdLineProcessor( sys.argv )
  268. if argv:
  269. - gdal2tiles = GDAL2Tiles( argv[1:] )
  270. - gdal2tiles.process()
  271. + gdal2tiles = GDAL2Tiles( argv[1:] ) # handle command line options
  272. +
  273. + p = multiprocessing.Process(target=worker_metadata, args=[argv])
  274. + p.start()
  275. + p.join()
  276. +
  277. + pool = multiprocessing.Pool()
  278. + processed_tiles = 0
  279. + print("Generating Base Tiles:")
  280. + for cpu in range(gdal2tiles.options.processes):
  281. + pool.apply_async(worker_base_tiles, [argv, cpu])
  282. + pool.close()
  283. + while len(multiprocessing.active_children()) != 0:
  284. + try:
  285. + total = queue.get(timeout=1)
  286. + processed_tiles += 1
  287. + gdal.TermProgress_nocb(processed_tiles / float(total))
  288. + sys.stdout.flush()
  289. + except:
  290. + pass
  291. + pool.join()
  292. +
  293. + processed_tiles = 0
  294. + print("Generating Overview Tiles:")
  295. + for tz in range(gdal2tiles.tmaxz-1, gdal2tiles.tminz-1, -1):
  296. + pool = multiprocessing.Pool()
  297. + for cpu in range(gdal2tiles.options.processes):
  298. + pool.apply_async(worker_overview_tiles, [argv, cpu, tz])
  299. + pool.close()
  300. + while len(multiprocessing.active_children()) != 0:
  301. + try:
  302. + total = queue.get(timeout=1)
  303. + processed_tiles += 1
  304. + gdal.TermProgress_nocb(processed_tiles / float(total))
  305. + sys.stdout.flush()
  306. + except:
  307. + pass
  308. + pool.join()
  309. +
  310. +
  311. +#############
  312. +# vim:noet
  313. +#############