修改缓存策略,提高读取性能

This commit is contained in:
zhouxin 2021-07-24 19:42:50 +08:00
parent ae620231fe
commit 453e91f87c
6 changed files with 59 additions and 36 deletions

View File

@ -30,6 +30,12 @@
<version>3.14.9</version> <version>3.14.9</version>
</dependency> </dependency>
<dependency>
<groupId>com.github.ben-manes.caffeine</groupId>
<artifactId>caffeine</artifactId>
<version>2.7.0</version>
</dependency>
<dependency> <dependency>
<groupId>org.springframework.boot</groupId> <groupId>org.springframework.boot</groupId>

View File

@ -7,6 +7,7 @@ import okhttp3.*;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.util.Assert; import org.springframework.util.Assert;
import org.springframework.util.StringUtils;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
@ -92,8 +93,12 @@ public class AliYunDriverClient {
} }
public InputStream download(String url) { public InputStream download(String url, String range) {
Request request = new Request.Builder().header("referer", "https://www.aliyundrive.com/").url(url).build(); Request.Builder builder = new Request.Builder().header("referer", "https://www.aliyundrive.com/");
if (StringUtils.hasLength(range)) {
builder.header("range", range);
}
Request request = builder.url(url).build();
Response response = null; Response response = null;
try { try {
response = okHttpClient.newCall(request).execute(); response = okHttpClient.newCall(request).execute();

View File

@ -3,6 +3,7 @@ package com.github.zxbu.webdavteambition.model;
public class DownloadRequest { public class DownloadRequest {
private String drive_id; private String drive_id;
private String file_id; private String file_id;
private Integer expire_sec = 14400;
public String getDrive_id() { public String getDrive_id() {
return drive_id; return drive_id;
@ -19,4 +20,12 @@ public class DownloadRequest {
public void setFile_id(String file_id) { public void setFile_id(String file_id) {
this.file_id = file_id; this.file_id = file_id;
} }
public Integer getExpire_sec() {
return expire_sec;
}
public void setExpire_sec(Integer expire_sec) {
this.expire_sec = expire_sec;
}
} }

View File

@ -2,6 +2,8 @@ package com.github.zxbu.webdavteambition.store;
import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
import com.github.zxbu.webdavteambition.client.AliYunDriverClient; import com.github.zxbu.webdavteambition.client.AliYunDriverClient;
import com.github.zxbu.webdavteambition.model.*; import com.github.zxbu.webdavteambition.model.*;
import com.github.zxbu.webdavteambition.model.result.TFile; import com.github.zxbu.webdavteambition.model.result.TFile;
@ -21,6 +23,7 @@ import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.util.*; import java.util.*;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
@Service @Service
public class AliYunDriverClientService { public class AliYunDriverClientService {
@ -29,7 +32,12 @@ public class AliYunDriverClientService {
private static String rootPath = "/"; private static String rootPath = "/";
private static int chunkSize = 10485760; // 10MB private static int chunkSize = 10485760; // 10MB
private TFile rootTFile = null; private TFile rootTFile = null;
private final ThreadLocal<Map<String, Set<TFile>>> tFilesCache = new ThreadLocal<>();
private static Cache<String, Set<TFile>> tFilesCache = Caffeine.newBuilder()
.initialCapacity(128)
.maximumSize(1024)
.expireAfterWrite(1, TimeUnit.MINUTES)
.build();
private final AliYunDriverClient client; private final AliYunDriverClient client;
@ -42,12 +50,7 @@ public class AliYunDriverClientService {
} }
public Set<TFile> getTFiles(String nodeId) { public Set<TFile> getTFiles(String nodeId) {
Map<String, Set<TFile>> map = tFilesCache.get(); Set<TFile> tFiles = tFilesCache.get(nodeId, key -> {
if (map == null) {
map = new ConcurrentHashMap<>();
tFilesCache.set(map);
}
Set<TFile> tFiles = map.computeIfAbsent(nodeId, key -> {
// 获取真实的文件列表 // 获取真实的文件列表
return getTFiles2(nodeId); return getTFiles2(nodeId);
}); });
@ -276,14 +279,14 @@ public class AliYunDriverClientService {
return getNodeIdByPath2(path); return getNodeIdByPath2(path);
} }
public InputStream download(String path) { public InputStream download(String path, String range) {
TFile file = getTFileByPath(path); TFile file = getTFileByPath(path);
DownloadRequest downloadRequest = new DownloadRequest(); DownloadRequest downloadRequest = new DownloadRequest();
downloadRequest.setDrive_id(client.getDriveId()); downloadRequest.setDrive_id(client.getDriveId());
downloadRequest.setFile_id(file.getFile_id()); downloadRequest.setFile_id(file.getFile_id());
String json = client.post("/file/get_download_url", downloadRequest); String json = client.post("/file/get_download_url", downloadRequest);
Object url = JsonUtil.getJsonNodeValue(json, "url"); Object url = JsonUtil.getJsonNodeValue(json, "url");
return client.download(url.toString()); return client.download(url.toString(), range);
} }
private TFile getNodeIdByPath2(String path) { private TFile getNodeIdByPath2(String path) {
@ -357,11 +360,7 @@ public class AliYunDriverClientService {
return path; return path;
} }
public void clearCache() { private void clearCache() {
Map<String, Set<TFile>> map = tFilesCache.get(); tFilesCache.invalidateAll();
if (map != null) {
map.clear();
}
} }
} }

View File

@ -36,31 +36,29 @@ public class AliYunDriverFileSystemStore implements IWebdavStore {
@Override @Override
public void destroy() { public void destroy() {
LOGGER.debug("destroy"); LOGGER.info("destroy");
} }
@Override @Override
public ITransaction begin(Principal principal, HttpServletRequest req, HttpServletResponse resp) { public ITransaction begin(Principal principal, HttpServletRequest req, HttpServletResponse resp) {
LOGGER.debug("begin"); LOGGER.info("begin");
aliYunDriverClientService.clearCache();
return new Transaction(principal, req, resp); return new Transaction(principal, req, resp);
} }
@Override @Override
public void checkAuthentication(ITransaction transaction) { public void checkAuthentication(ITransaction transaction) {
LOGGER.debug("checkAuthentication"); LOGGER.info("checkAuthentication");
} }
@Override @Override
public void commit(ITransaction transaction) { public void commit(ITransaction transaction) {
aliYunDriverClientService.clearCache(); LOGGER.info("commit");
LOGGER.debug("commit");
} }
@Override @Override
public void rollback(ITransaction transaction) { public void rollback(ITransaction transaction) {
LOGGER.debug("rollback"); LOGGER.info("rollback");
} }
@ -79,8 +77,10 @@ public class AliYunDriverFileSystemStore implements IWebdavStore {
@Override @Override
public InputStream getResourceContent(ITransaction transaction, String resourceUri) { public InputStream getResourceContent(ITransaction transaction, String resourceUri) {
LOGGER.debug("getResourceContent: {}", resourceUri); LOGGER.info("getResourceContent: {}", resourceUri);
return aliYunDriverClientService.download(resourceUri); String range = transaction.getRequest().getHeader("range");
return aliYunDriverClientService.download(resourceUri, range);
} }
@Override @Override
@ -113,7 +113,7 @@ public class AliYunDriverFileSystemStore implements IWebdavStore {
@Override @Override
public String[] getChildrenNames(ITransaction transaction, String folderUri) { public String[] getChildrenNames(ITransaction transaction, String folderUri) {
LOGGER.debug("getChildrenNames: {}", folderUri); LOGGER.info("getChildrenNames: {}", folderUri);
TFile tFile = aliYunDriverClientService.getTFileByPath(folderUri); TFile tFile = aliYunDriverClientService.getTFileByPath(folderUri);
if (tFile.getType().equals(FileType.file.name())) { if (tFile.getType().equals(FileType.file.name())) {
return new String[0]; return new String[0];
@ -126,7 +126,7 @@ public class AliYunDriverFileSystemStore implements IWebdavStore {
@Override @Override
public long getResourceLength(ITransaction transaction, String path) { public long getResourceLength(ITransaction transaction, String path) {
LOGGER.debug("getResourceLength: {}", path); LOGGER.info("getResourceLength: {}", path);
TFile tFile = aliYunDriverClientService.getTFileByPath(path); TFile tFile = aliYunDriverClientService.getTFileByPath(path);
if (tFile == null || tFile.getSize() == null) { if (tFile == null || tFile.getSize() == null) {
return 384; return 384;
@ -164,7 +164,7 @@ public class AliYunDriverFileSystemStore implements IWebdavStore {
public StoredObject getStoredObject(ITransaction transaction, String uri) { public StoredObject getStoredObject(ITransaction transaction, String uri) {
LOGGER.debug("getStoredObject: {}", uri); LOGGER.info("getStoredObject: {}", uri);
TFile tFile = aliYunDriverClientService.getTFileByPath(uri); TFile tFile = aliYunDriverClientService.getTFileByPath(uri);
if (tFile != null) { if (tFile != null) {
StoredObject so = new StoredObject(); StoredObject so = new StoredObject();

View File

@ -61,27 +61,31 @@ public class DoGet extends DoHead {
OutputStream out = resp.getOutputStream(); OutputStream out = resp.getOutputStream();
InputStream in = _store.getResourceContent(transaction, path); InputStream in = _store.getResourceContent(transaction, path);
try { try {
int read = -1; if (in != null) {
byte[] copyBuffer = new byte[BUF_SIZE]; int read = -1;
byte[] copyBuffer = new byte[BUF_SIZE];
while ((read = in.read(copyBuffer, 0, copyBuffer.length)) != -1) { while ((read = in.read(copyBuffer, 0, copyBuffer.length)) != -1) {
out.write(copyBuffer, 0, read); out.write(copyBuffer, 0, read);
}
} }
} finally { } finally {
// flushing causes a IOE if a file is opened on the webserver // flushing causes a IOE if a file is opened on the webserver
// client disconnected before server finished sending response // client disconnected before server finished sending response
try { try {
in.close(); if (in != null) {
in.close();
}
} catch (Exception e) { } catch (Exception e) {
LOG.warn("Closing InputStream causes Exception!\n" LOG.warn("Closing InputStream causes Exception!\n"
+ e.toString()); ,e);
} }
try { try {
out.flush(); out.flush();
out.close(); out.close();
} catch (Exception e) { } catch (Exception e) {
LOG.warn("Flushing OutputStream causes Exception!\n" LOG.warn("Flushing OutputStream causes Exception!\n"
+ e.toString()); ,e);
} }
} }
} catch (Exception e) { } catch (Exception e) {