001// License: GPL. For details, see LICENSE file. 002package org.openstreetmap.josm.data.imagery; 003 004import java.awt.Graphics2D; 005import java.awt.image.BufferedImage; 006import java.io.BufferedOutputStream; 007import java.io.File; 008import java.io.FileInputStream; 009import java.io.FileNotFoundException; 010import java.io.FileOutputStream; 011import java.io.IOException; 012import java.io.InputStream; 013import java.io.OutputStream; 014import java.lang.ref.SoftReference; 015import java.net.URLConnection; 016import java.util.ArrayList; 017import java.util.Calendar; 018import java.util.Collections; 019import java.util.Comparator; 020import java.util.HashMap; 021import java.util.HashSet; 022import java.util.Iterator; 023import java.util.List; 024import java.util.Map; 025import java.util.Properties; 026import java.util.Set; 027 028import javax.imageio.ImageIO; 029import javax.xml.bind.JAXBContext; 030import javax.xml.bind.Marshaller; 031import javax.xml.bind.Unmarshaller; 032 033import org.openstreetmap.josm.Main; 034import org.openstreetmap.josm.data.ProjectionBounds; 035import org.openstreetmap.josm.data.SystemOfMeasurement; 036import org.openstreetmap.josm.data.coor.EastNorth; 037import org.openstreetmap.josm.data.coor.LatLon; 038import org.openstreetmap.josm.data.imagery.types.EntryType; 039import org.openstreetmap.josm.data.imagery.types.ProjectionType; 040import org.openstreetmap.josm.data.imagery.types.WmsCacheType; 041import org.openstreetmap.josm.data.preferences.StringProperty; 042import org.openstreetmap.josm.data.projection.Projection; 043import org.openstreetmap.josm.gui.layer.WMSLayer; 044import org.openstreetmap.josm.tools.ImageProvider; 045import org.openstreetmap.josm.tools.Utils; 046import org.openstreetmap.josm.tools.date.DateUtils; 047 048public class WmsCache { 049 //TODO Property for maximum cache size 050 //TODO Property for maximum age of tile, automatically remove old tiles 051 //TODO Measure time for partially loading from cache, compare with time to download tile. If slower, disable partial cache 052 //TODO Do loading from partial cache and downloading at the same time, don't wait for partial cache to load 053 054 private static final StringProperty PROP_CACHE_PATH = new StringProperty("imagery.wms-cache.path", "wms"); 055 private static final String INDEX_FILENAME = "index.xml"; 056 private static final String LAYERS_INDEX_FILENAME = "layers.properties"; 057 058 private static class CacheEntry { 059 final double pixelPerDegree; 060 final double east; 061 final double north; 062 final ProjectionBounds bounds; 063 final String filename; 064 065 long lastUsed; 066 long lastModified; 067 068 CacheEntry(double pixelPerDegree, double east, double north, int tileSize, String filename) { 069 this.pixelPerDegree = pixelPerDegree; 070 this.east = east; 071 this.north = north; 072 this.bounds = new ProjectionBounds(east, north, east + tileSize / pixelPerDegree, north + tileSize / pixelPerDegree); 073 this.filename = filename; 074 } 075 076 @Override 077 public String toString() { 078 return "CacheEntry [pixelPerDegree=" + pixelPerDegree + ", east=" + east + ", north=" + north + ", bounds=" 079 + bounds + ", filename=" + filename + ", lastUsed=" + lastUsed + ", lastModified=" + lastModified 080 + "]"; 081 } 082 } 083 084 private static class ProjectionEntries { 085 final String projection; 086 final String cacheDirectory; 087 final List<CacheEntry> entries = new ArrayList<>(); 088 089 ProjectionEntries(String projection, String cacheDirectory) { 090 this.projection = projection; 091 this.cacheDirectory = cacheDirectory; 092 } 093 } 094 095 private final Map<String, ProjectionEntries> entries = new HashMap<>(); 096 private final File cacheDir; 097 private final int tileSize; // Should be always 500 098 private int totalFileSize; 099 private boolean totalFileSizeDirty; // Some file was missing - size needs to be recalculated 100 // No need for hashCode/equals on CacheEntry, object identity is enough. Comparing by values can lead to error - CacheEntry for wrong projection could be found 101 private Map<CacheEntry, SoftReference<BufferedImage>> memoryCache = new HashMap<>(); 102 private Set<ProjectionBounds> areaToCache; 103 104 protected String cacheDirPath() { 105 String cPath = PROP_CACHE_PATH.get(); 106 if (!(new File(cPath).isAbsolute())) { 107 cPath = Main.pref.getCacheDirectory() + File.separator + cPath; 108 } 109 return cPath; 110 } 111 112 public WmsCache(String url, int tileSize) { 113 File globalCacheDir = new File(cacheDirPath()); 114 globalCacheDir.mkdirs(); 115 cacheDir = new File(globalCacheDir, getCacheDirectory(url)); 116 cacheDir.mkdirs(); 117 this.tileSize = tileSize; 118 } 119 120 private String getCacheDirectory(String url) { 121 String cacheDirName = null; 122 Properties layersIndex = new Properties(); 123 File layerIndexFile = new File(cacheDirPath(), LAYERS_INDEX_FILENAME); 124 try (InputStream fis = new FileInputStream(layerIndexFile)) { 125 layersIndex.load(fis); 126 } catch (FileNotFoundException e) { 127 Main.error("Unable to load layers index for wms cache (file " + layerIndexFile + " not found)"); 128 } catch (IOException e) { 129 Main.error("Unable to load layers index for wms cache"); 130 Main.error(e); 131 } 132 133 for (Object propKey: layersIndex.keySet()) { 134 String s = (String)propKey; 135 if (url.equals(layersIndex.getProperty(s))) { 136 cacheDirName = s; 137 break; 138 } 139 } 140 141 if (cacheDirName == null) { 142 int counter = 0; 143 while (true) { 144 counter++; 145 if (!layersIndex.keySet().contains(String.valueOf(counter))) { 146 break; 147 } 148 } 149 cacheDirName = String.valueOf(counter); 150 layersIndex.setProperty(cacheDirName, url); 151 try (OutputStream fos = new FileOutputStream(layerIndexFile)) { 152 layersIndex.store(fos, ""); 153 } catch (IOException e) { 154 Main.error("Unable to save layer index for wms cache"); 155 Main.error(e); 156 } 157 } 158 159 return cacheDirName; 160 } 161 162 private ProjectionEntries getProjectionEntries(Projection projection) { 163 return getProjectionEntries(projection.toCode(), projection.getCacheDirectoryName()); 164 } 165 166 private ProjectionEntries getProjectionEntries(String projection, String cacheDirectory) { 167 ProjectionEntries result = entries.get(projection); 168 if (result == null) { 169 result = new ProjectionEntries(projection, cacheDirectory); 170 entries.put(projection, result); 171 } 172 173 return result; 174 } 175 176 public synchronized void loadIndex() { 177 File indexFile = new File(cacheDir, INDEX_FILENAME); 178 try { 179 JAXBContext context = JAXBContext.newInstance( 180 WmsCacheType.class.getPackage().getName(), 181 WmsCacheType.class.getClassLoader()); 182 Unmarshaller unmarshaller = context.createUnmarshaller(); 183 WmsCacheType cacheEntries; 184 try (InputStream is = new FileInputStream(indexFile)) { 185 cacheEntries = (WmsCacheType)unmarshaller.unmarshal(is); 186 } 187 totalFileSize = cacheEntries.getTotalFileSize(); 188 if (cacheEntries.getTileSize() != tileSize) { 189 Main.info("Cache created with different tileSize, cache will be discarded"); 190 return; 191 } 192 for (ProjectionType projectionType: cacheEntries.getProjection()) { 193 ProjectionEntries projection = getProjectionEntries(projectionType.getName(), projectionType.getCacheDirectory()); 194 for (EntryType entry: projectionType.getEntry()) { 195 CacheEntry ce = new CacheEntry(entry.getPixelPerDegree(), entry.getEast(), entry.getNorth(), tileSize, entry.getFilename()); 196 ce.lastUsed = entry.getLastUsed().getTimeInMillis(); 197 ce.lastModified = entry.getLastModified().getTimeInMillis(); 198 projection.entries.add(ce); 199 } 200 } 201 } catch (Exception e) { 202 if (indexFile.exists()) { 203 Main.error(e); 204 Main.info("Unable to load index for wms-cache, new file will be created"); 205 } else { 206 Main.info("Index for wms-cache doesn't exist, new file will be created"); 207 } 208 } 209 210 removeNonReferencedFiles(); 211 } 212 213 private void removeNonReferencedFiles() { 214 215 Set<String> usedProjections = new HashSet<>(); 216 217 for (ProjectionEntries projectionEntries: entries.values()) { 218 219 usedProjections.add(projectionEntries.cacheDirectory); 220 221 File projectionDir = new File(cacheDir, projectionEntries.cacheDirectory); 222 if (projectionDir.exists()) { 223 Set<String> referencedFiles = new HashSet<>(); 224 225 for (CacheEntry ce: projectionEntries.entries) { 226 referencedFiles.add(ce.filename); 227 } 228 229 for (File file: projectionDir.listFiles()) { 230 if (!referencedFiles.contains(file.getName())) { 231 file.delete(); 232 } 233 } 234 } 235 } 236 237 for (File projectionDir: cacheDir.listFiles()) { 238 if (projectionDir.isDirectory() && !usedProjections.contains(projectionDir.getName())) { 239 Utils.deleteDirectory(projectionDir); 240 } 241 } 242 } 243 244 private int calculateTotalFileSize() { 245 int result = 0; 246 for (ProjectionEntries projectionEntries: entries.values()) { 247 Iterator<CacheEntry> it = projectionEntries.entries.iterator(); 248 while (it.hasNext()) { 249 CacheEntry entry = it.next(); 250 File imageFile = getImageFile(projectionEntries, entry); 251 if (!imageFile.exists()) { 252 it.remove(); 253 } else { 254 result += imageFile.length(); 255 } 256 } 257 } 258 return result; 259 } 260 261 public synchronized void saveIndex() { 262 WmsCacheType index = new WmsCacheType(); 263 264 if (totalFileSizeDirty) { 265 totalFileSize = calculateTotalFileSize(); 266 } 267 268 index.setTileSize(tileSize); 269 index.setTotalFileSize(totalFileSize); 270 for (ProjectionEntries projectionEntries: entries.values()) { 271 if (!projectionEntries.entries.isEmpty()) { 272 ProjectionType projectionType = new ProjectionType(); 273 projectionType.setName(projectionEntries.projection); 274 projectionType.setCacheDirectory(projectionEntries.cacheDirectory); 275 index.getProjection().add(projectionType); 276 for (CacheEntry ce: projectionEntries.entries) { 277 EntryType entry = new EntryType(); 278 entry.setPixelPerDegree(ce.pixelPerDegree); 279 entry.setEast(ce.east); 280 entry.setNorth(ce.north); 281 Calendar c = Calendar.getInstance(); 282 c.setTimeInMillis(ce.lastUsed); 283 entry.setLastUsed(c); 284 c = Calendar.getInstance(); 285 c.setTimeInMillis(ce.lastModified); 286 entry.setLastModified(c); 287 entry.setFilename(ce.filename); 288 projectionType.getEntry().add(entry); 289 } 290 } 291 } 292 try { 293 JAXBContext context = JAXBContext.newInstance( 294 WmsCacheType.class.getPackage().getName(), 295 WmsCacheType.class.getClassLoader()); 296 Marshaller marshaller = context.createMarshaller(); 297 try (OutputStream fos = new FileOutputStream(new File(cacheDir, INDEX_FILENAME))) { 298 marshaller.marshal(index, fos); 299 } 300 } catch (Exception e) { 301 Main.error("Failed to save wms-cache file"); 302 Main.error(e); 303 } 304 } 305 306 private File getImageFile(ProjectionEntries projection, CacheEntry entry) { 307 return new File(cacheDir, projection.cacheDirectory + "/" + entry.filename); 308 } 309 310 private BufferedImage loadImage(ProjectionEntries projectionEntries, CacheEntry entry, boolean enforceTransparency) throws IOException { 311 synchronized (this) { 312 entry.lastUsed = System.currentTimeMillis(); 313 314 SoftReference<BufferedImage> memCache = memoryCache.get(entry); 315 if (memCache != null) { 316 BufferedImage result = memCache.get(); 317 if (result != null) { 318 if (enforceTransparency == ImageProvider.isTransparencyForced(result)) { 319 return result; 320 } else if (Main.isDebugEnabled()) { 321 Main.debug("Skipping "+entry+" from memory cache (transparency enforcement)"); 322 } 323 } 324 } 325 } 326 327 try { 328 // Reading can't be in synchronized section, it's too slow 329 BufferedImage result = ImageProvider.read(getImageFile(projectionEntries, entry), true, enforceTransparency); 330 synchronized (this) { 331 if (result == null) { 332 projectionEntries.entries.remove(entry); 333 totalFileSizeDirty = true; 334 } 335 return result; 336 } 337 } catch (IOException e) { 338 synchronized (this) { 339 projectionEntries.entries.remove(entry); 340 totalFileSizeDirty = true; 341 throw e; 342 } 343 } 344 } 345 346 private CacheEntry findEntry(ProjectionEntries projectionEntries, double pixelPerDegree, double east, double north) { 347 for (CacheEntry entry: projectionEntries.entries) { 348 if (entry.pixelPerDegree == pixelPerDegree && entry.east == east && entry.north == north) 349 return entry; 350 } 351 return null; 352 } 353 354 public synchronized boolean hasExactMatch(Projection projection, double pixelPerDegree, double east, double north) { 355 ProjectionEntries projectionEntries = getProjectionEntries(projection); 356 CacheEntry entry = findEntry(projectionEntries, pixelPerDegree, east, north); 357 return (entry != null); 358 } 359 360 public BufferedImage getExactMatch(Projection projection, double pixelPerDegree, double east, double north) { 361 CacheEntry entry = null; 362 ProjectionEntries projectionEntries = null; 363 synchronized (this) { 364 projectionEntries = getProjectionEntries(projection); 365 entry = findEntry(projectionEntries, pixelPerDegree, east, north); 366 } 367 if (entry != null) { 368 try { 369 return loadImage(projectionEntries, entry, WMSLayer.PROP_ALPHA_CHANNEL.get()); 370 } catch (IOException e) { 371 Main.error("Unable to load file from wms cache"); 372 Main.error(e); 373 return null; 374 } 375 } 376 return null; 377 } 378 379 public BufferedImage getPartialMatch(Projection projection, double pixelPerDegree, double east, double north) { 380 ProjectionEntries projectionEntries; 381 List<CacheEntry> matches; 382 synchronized (this) { 383 matches = new ArrayList<>(); 384 385 double minPPD = pixelPerDegree / 5; 386 double maxPPD = pixelPerDegree * 5; 387 projectionEntries = getProjectionEntries(projection); 388 389 double size2 = tileSize / pixelPerDegree; 390 double border = tileSize * 0.01; // Make sure not to load neighboring tiles that intersects this tile only slightly 391 ProjectionBounds bounds = new ProjectionBounds(east + border, north + border, 392 east + size2 - border, north + size2 - border); 393 394 //TODO Do not load tile if it is completely overlapped by other tile with better ppd 395 for (CacheEntry entry: projectionEntries.entries) { 396 if (entry.pixelPerDegree >= minPPD && entry.pixelPerDegree <= maxPPD && entry.bounds.intersects(bounds)) { 397 entry.lastUsed = System.currentTimeMillis(); 398 matches.add(entry); 399 } 400 } 401 402 if (matches.isEmpty()) 403 return null; 404 405 Collections.sort(matches, new Comparator<CacheEntry>() { 406 @Override 407 public int compare(CacheEntry o1, CacheEntry o2) { 408 return Double.compare(o2.pixelPerDegree, o1.pixelPerDegree); 409 } 410 }); 411 } 412 413 // Use alpha layer only when enabled on wms layer 414 boolean alpha = WMSLayer.PROP_ALPHA_CHANNEL.get(); 415 BufferedImage result = new BufferedImage(tileSize, tileSize, 416 alpha ? BufferedImage.TYPE_4BYTE_ABGR : BufferedImage.TYPE_3BYTE_BGR); 417 Graphics2D g = result.createGraphics(); 418 419 boolean drawAtLeastOnce = false; 420 Map<CacheEntry, SoftReference<BufferedImage>> localCache = new HashMap<>(); 421 for (CacheEntry ce: matches) { 422 BufferedImage img; 423 try { 424 // Enforce transparency only when alpha enabled on wms layer too 425 img = loadImage(projectionEntries, ce, alpha); 426 localCache.put(ce, new SoftReference<>(img)); 427 } catch (IOException e) { 428 continue; 429 } 430 431 drawAtLeastOnce = true; 432 433 int xDiff = (int)((ce.east - east) * pixelPerDegree); 434 int yDiff = (int)((ce.north - north) * pixelPerDegree); 435 int size = (int)(pixelPerDegree / ce.pixelPerDegree * tileSize); 436 437 int x = xDiff; 438 int y = -size + tileSize - yDiff; 439 440 g.drawImage(img, x, y, size, size, null); 441 } 442 443 if (drawAtLeastOnce) { 444 synchronized (this) { 445 memoryCache.putAll(localCache); 446 } 447 return result; 448 } else 449 return null; 450 } 451 452 private String generateFileName(ProjectionEntries projectionEntries, double pixelPerDegree, Projection projection, double east, double north, String mimeType) { 453 LatLon ll1 = projection.eastNorth2latlon(new EastNorth(east, north)); 454 LatLon ll2 = projection.eastNorth2latlon(new EastNorth(east + 100 / pixelPerDegree, north)); 455 LatLon ll3 = projection.eastNorth2latlon(new EastNorth(east + tileSize / pixelPerDegree, north + tileSize / pixelPerDegree)); 456 457 double deltaLat = Math.abs(ll3.lat() - ll1.lat()); 458 double deltaLon = Math.abs(ll3.lon() - ll1.lon()); 459 int precisionLat = Math.max(0, -(int)Math.ceil(Math.log10(deltaLat)) + 1); 460 int precisionLon = Math.max(0, -(int)Math.ceil(Math.log10(deltaLon)) + 1); 461 462 String zoom = SystemOfMeasurement.METRIC.getDistText(ll1.greatCircleDistance(ll2)); 463 String extension = "dat"; 464 if (mimeType != null) { 465 switch(mimeType) { 466 case "image/jpeg": 467 case "image/jpg": 468 extension = "jpg"; 469 break; 470 case "image/png": 471 extension = "png"; 472 break; 473 case "image/gif": 474 extension = "gif"; 475 break; 476 default: 477 Main.warn("Unrecognized MIME type: "+mimeType); 478 } 479 } 480 481 int counter = 0; 482 FILENAME_LOOP: 483 while (true) { 484 String result = String.format("%s_%." + precisionLat + "f_%." + precisionLon +"f%s.%s", zoom, ll1.lat(), ll1.lon(), counter==0?"":"_" + counter, extension); 485 for (CacheEntry entry: projectionEntries.entries) { 486 if (entry.filename.equals(result)) { 487 counter++; 488 continue FILENAME_LOOP; 489 } 490 } 491 return result; 492 } 493 } 494 495 /** 496 * 497 * @param img Used only when overlapping is used, when not used, used raw from imageData 498 * @param imageData 499 * @param projection 500 * @param pixelPerDegree 501 * @param east 502 * @param north 503 * @throws IOException 504 */ 505 public synchronized void saveToCache(BufferedImage img, InputStream imageData, Projection projection, double pixelPerDegree, double east, double north) throws IOException { 506 ProjectionEntries projectionEntries = getProjectionEntries(projection); 507 CacheEntry entry = findEntry(projectionEntries, pixelPerDegree, east, north); 508 File imageFile; 509 if (entry == null) { 510 511 String mimeType; 512 if (img != null) { 513 mimeType = "image/png"; 514 } else { 515 mimeType = URLConnection.guessContentTypeFromStream(imageData); 516 } 517 entry = new CacheEntry(pixelPerDegree, east, north, tileSize,generateFileName(projectionEntries, pixelPerDegree, projection, east, north, mimeType)); 518 entry.lastUsed = System.currentTimeMillis(); 519 entry.lastModified = entry.lastUsed; 520 projectionEntries.entries.add(entry); 521 imageFile = getImageFile(projectionEntries, entry); 522 } else { 523 imageFile = getImageFile(projectionEntries, entry); 524 totalFileSize -= imageFile.length(); 525 } 526 527 imageFile.getParentFile().mkdirs(); 528 529 if (img != null) { 530 BufferedImage copy = new BufferedImage(tileSize, tileSize, img.getType()); 531 copy.createGraphics().drawImage(img, 0, 0, tileSize, tileSize, 0, img.getHeight() - tileSize, tileSize, img.getHeight(), null); 532 ImageIO.write(copy, "png", imageFile); 533 totalFileSize += imageFile.length(); 534 } else { 535 try (OutputStream os = new BufferedOutputStream(new FileOutputStream(imageFile))) { 536 totalFileSize += Utils.copyStream(imageData, os); 537 } 538 } 539 } 540 541 public synchronized void cleanSmallFiles(int size) { 542 for (ProjectionEntries projectionEntries: entries.values()) { 543 Iterator<CacheEntry> it = projectionEntries.entries.iterator(); 544 while (it.hasNext()) { 545 File file = getImageFile(projectionEntries, it.next()); 546 long length = file.length(); 547 if (length <= size) { 548 if (length == 0) { 549 totalFileSizeDirty = true; // File probably doesn't exist 550 } 551 totalFileSize -= size; 552 file.delete(); 553 it.remove(); 554 } 555 } 556 } 557 } 558 559 public static String printDate(Calendar c) { 560 return DateUtils.newIsoDateFormat().format(c.getTime()); 561 } 562 563 private boolean isInsideAreaToCache(CacheEntry cacheEntry) { 564 for (ProjectionBounds b: areaToCache) { 565 if (cacheEntry.bounds.intersects(b)) 566 return true; 567 } 568 return false; 569 } 570 571 public synchronized void setAreaToCache(Set<ProjectionBounds> areaToCache) { 572 this.areaToCache = areaToCache; 573 Iterator<CacheEntry> it = memoryCache.keySet().iterator(); 574 while (it.hasNext()) { 575 if (!isInsideAreaToCache(it.next())) { 576 it.remove(); 577 } 578 } 579 } 580}