1. Project Clover database Tue Dec 20 2016 21:24:09 CET
  2. Package org.xwiki.extension.repository.internal.core

File DefaultCoreExtensionScanner.java

 

Coverage histogram

../../../../../../img/srcFileCovDistChart9.png
38% of files have more coverage

Code metrics

76
216
16
1
622
433
70
0.32
13.5
16
4.38

Classes

Class Line # Actions
DefaultCoreExtensionScanner 80 216 0% 70 42
0.863636486.4%
 

Contributing tests

This file is covered by 1 test. .

Source view

1    /*
2    * See the NOTICE file distributed with this work for additional
3    * information regarding copyright ownership.
4    *
5    * This is free software; you can redistribute it and/or modify it
6    * under the terms of the GNU Lesser General Public License as
7    * published by the Free Software Foundation; either version 2.1 of
8    * the License, or (at your option) any later version.
9    *
10    * This software is distributed in the hope that it will be useful,
11    * but WITHOUT ANY WARRANTY; without even the implied warranty of
12    * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13    * Lesser General Public License for more details.
14    *
15    * You should have received a copy of the GNU Lesser General Public
16    * License along with this software; if not, write to the Free
17    * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
18    * 02110-1301 USA, or see the FSF site: http://www.fsf.org.
19    */
20    package org.xwiki.extension.repository.internal.core;
21   
22    import java.io.IOException;
23    import java.io.InputStream;
24    import java.net.MalformedURLException;
25    import java.net.URL;
26    import java.util.ArrayList;
27    import java.util.Collection;
28    import java.util.HashMap;
29    import java.util.HashSet;
30    import java.util.Iterator;
31    import java.util.Map;
32    import java.util.Set;
33    import java.util.jar.Attributes;
34    import java.util.jar.Manifest;
35    import java.util.regex.Matcher;
36   
37    import javax.inject.Inject;
38    import javax.inject.Singleton;
39   
40    import org.apache.commons.io.IOUtils;
41    import org.apache.commons.lang3.StringUtils;
42    import org.apache.commons.lang3.exception.ExceptionUtils;
43    import org.apache.maven.model.Dependency;
44    import org.apache.maven.model.Model;
45    import org.apache.maven.model.io.xpp3.MavenXpp3Reader;
46    import org.reflections.Reflections;
47    import org.reflections.scanners.ResourcesScanner;
48    import org.reflections.util.ClasspathHelper;
49    import org.reflections.util.ConfigurationBuilder;
50    import org.reflections.util.FilterBuilder;
51    import org.slf4j.Logger;
52    import org.slf4j.LoggerFactory;
53    import org.xwiki.component.annotation.Component;
54    import org.xwiki.component.manager.ComponentLifecycleException;
55    import org.xwiki.component.phase.Disposable;
56    import org.xwiki.environment.Environment;
57    import org.xwiki.extension.Extension;
58    import org.xwiki.extension.ExtensionDependency;
59    import org.xwiki.extension.ExtensionId;
60    import org.xwiki.extension.ResolveException;
61    import org.xwiki.extension.internal.PathUtils;
62    import org.xwiki.extension.internal.maven.MavenExtension;
63    import org.xwiki.extension.internal.maven.MavenExtensionDependency;
64    import org.xwiki.extension.internal.maven.MavenUtils;
65    import org.xwiki.extension.repository.ExtensionRepositoryManager;
66    import org.xwiki.extension.repository.internal.ExtensionSerializer;
67    import org.xwiki.extension.version.Version;
68    import org.xwiki.properties.ConverterManager;
69   
70    import com.google.common.base.Predicates;
71   
72    /**
73    * Scan jars to find core extensions.
74    *
75    * @version $Id: b51338e60ee7c37cc77cd7155ed78ffd9119e265 $
76    * @since 4.0M1
77    */
78    @Component
79    @Singleton
 
80    public class DefaultCoreExtensionScanner implements CoreExtensionScanner, Disposable
81    {
82    /**
83    * Logger to use to log shutdown information (opposite of initialization).
84    */
85    private static final Logger SHUTDOWN_LOGGER = LoggerFactory.getLogger("org.xwiki.shutdown");
86   
87    /**
88    * The logger to log.
89    */
90    @Inject
91    private Logger logger;
92   
93    /**
94    * Used to resolve found core extensions.
95    */
96    @Inject
97    private ExtensionRepositoryManager repositoryManager;
98   
99    /**
100    * Used to parse some custom properties.
101    */
102    @Inject
103    private ConverterManager converter;
104   
105    @Inject
106    private Environment environment;
107   
108    @Inject
109    private CoreExtensionCache cache;
110   
111    @Inject
112    private ExtensionSerializer parser;
113   
114    private boolean shouldStop;
115   
 
116  87 toggle @Override
117    public void dispose() throws ComponentLifecycleException
118    {
119  87 this.shouldStop = true;
120    }
121   
 
122  1539 toggle private Dependency toDependency(String id, String version, String type) throws ResolveException
123    {
124  1539 Matcher matcher = MavenUtils.PARSER_ID.matcher(id);
125  1539 if (!matcher.matches()) {
126  0 throw new ResolveException("Bad id [" + id + "], expected format is <groupId>:<artifactId>[:<classifier>]");
127    }
128   
129  1539 Dependency dependency = new Dependency();
130   
131  1539 dependency.setGroupId(matcher.group(1));
132  1539 dependency.setArtifactId(matcher.group(2));
133  1539 if (matcher.group(4) != null) {
134  68 dependency.setClassifier(StringUtils.defaultString(matcher.group(4), ""));
135    }
136   
137  1539 if (version != null) {
138  1539 dependency.setVersion(version);
139    }
140   
141  1539 if (type != null) {
142  0 dependency.setType(type);
143    }
144   
145  1539 return dependency;
146    }
147   
 
148  22100 toggle private String getArtifactId(DefaultCoreExtension extension) throws ResolveException
149    {
150  22100 String artifactId;
151   
152  22100 if (extension instanceof MavenExtension) {
153  10361 artifactId = ((MavenExtension) extension).getMavenArtifactId();
154    } else {
155  11739 Matcher matcher = MavenUtils.PARSER_ID.matcher(extension.getId().getId());
156  11739 if (!matcher.matches()) {
157  0 throw new ResolveException("Bad id " + extension.getId().getId()
158    + ", expected format is <groupId>:<artifactId>[:<classifier>]");
159    }
160  11739 artifactId = matcher.group(2);
161    }
162   
163  22100 return artifactId;
164    }
165   
 
166  11467 toggle private DefaultCoreExtension getCoreExension(URL jarURL, URL descriptorURL,
167    DefaultCoreExtensionRepository repository) throws Exception
168    {
169  11467 DefaultCoreExtension coreExtension = this.cache.getExtension(repository, descriptorURL);
170   
171  11467 if (coreExtension != null && coreExtension.getDescriptorURL().equals(descriptorURL)) {
172  600 return coreExtension;
173    }
174   
175  10867 return parseMavenPom(jarURL, descriptorURL, repository);
176    }
177   
 
178  10899 toggle private DefaultCoreExtension parseMavenPom(URL jarURL, URL descriptorURL, DefaultCoreExtensionRepository repository)
179    throws Exception
180    {
181  10899 InputStream descriptorStream = descriptorURL.openStream();
182  10899 try {
183  10899 MavenXpp3Reader reader = new MavenXpp3Reader();
184  10899 Model mavenModel = reader.read(descriptorStream);
185   
186    // Resolve Maven variables in critical places
187  10899 MavenUtils.resolveVariables(mavenModel);
188   
189  10899 Extension mavenExtension = this.converter.convert(Extension.class, mavenModel);
190   
191  10899 DefaultCoreExtension coreExtension = new MavenCoreExtension(repository, jarURL, mavenExtension);
192  10899 coreExtension.setDescriptorURL(descriptorURL);
193   
194    // If no parent is provided no need to resolve it to get more details
195  10899 if (mavenModel.getParent() == null) {
196  257 this.cache.store(coreExtension);
197  257 coreExtension.setComplete(true);
198    }
199   
200  10899 return coreExtension;
201    } finally {
202  10899 IOUtils.closeQuietly(descriptorStream);
203    }
204    }
205   
 
206  2 toggle @Override
207    public void updateExtensions(Collection<DefaultCoreExtension> extensions)
208    {
209  2 for (DefaultCoreExtension extension : extensions) {
210    // If XWiki is stopping before this is finished then we need to exit.
211  805 if (this.shouldStop) {
212  0 SHUTDOWN_LOGGER.debug("Aborting Extension Update as XWiki is stopping");
213  0 break;
214    }
215   
216  805 if (!extension.isComplete()) {
217  39 try {
218  39 Extension remoteExtension = this.repositoryManager.resolve(extension.getId());
219   
220  19 extension.set(remoteExtension);
221  19 extension.setComplete(true);
222   
223    // Cache it
224  19 if (extension.getDescriptorURL() != null) {
225  17 this.cache.store(extension);
226    }
227    } catch (ResolveException e) {
228  20 this.logger.debug("Can't find remote extension with id [{}]", extension.getId(), e);
229    } catch (Exception e) {
230  0 this.logger.warn("Failed to update core extension [{}]: [{}]", extension.getId(),
231    ExceptionUtils.getRootCauseMessage(e), e);
232    }
233    }
234    }
235    }
236   
 
237  87 toggle @Override
238    public Map<String, DefaultCoreExtension> loadExtensions(DefaultCoreExtensionRepository repository)
239    {
240  87 Map<String, DefaultCoreExtension> extensions = new HashMap<>();
241   
242  87 loadExtensionsFromClassloaders(extensions, repository);
243   
244  87 return extensions;
245    }
246   
 
247  87 toggle @Override
248    public DefaultCoreExtension loadEnvironmentExtension(DefaultCoreExtensionRepository repository)
249    {
250    //////////
251    // XED
252   
253  87 URL xedURL = this.environment.getResource("/META-INF/extension.xed");
254  87 if (xedURL != null) {
255  0 try (InputStream xedStream = this.environment.getResourceAsStream("/META-INF/extension.xed")) {
256  0 return this.parser.loadCoreExtensionDescriptor(repository, null, xedStream);
257    } catch (Exception e) {
258  0 this.logger.error("Failed to load [{}] descriptor file", xedURL, e);
259    }
260    }
261   
262    //////////
263    // MAVEN
264   
265  87 InputStream manifestString = this.environment.getResourceAsStream("/META-INF/MANIFEST.MF");
266   
267  87 if (manifestString != null) {
268    // Probably running in an application server
269  32 try {
270  32 Manifest manifest = new Manifest(manifestString);
271   
272  32 Attributes manifestAttributes = manifest.getMainAttributes();
273  32 String extensionId = manifestAttributes.getValue(MavenUtils.MF_EXTENSION_ID);
274   
275  32 if (extensionId != null) {
276  32 String[] mavenId = StringUtils.split(extensionId, ':');
277   
278  32 String descriptorPath = String.format("/META-INF/maven/%s/%s/pom.xml", mavenId[0], mavenId[1]);
279   
280  32 URL descriptorURL = this.environment.getResource(descriptorPath);
281   
282  32 if (descriptorURL != null) {
283  32 try {
284  32 DefaultCoreExtension coreExtension =
285    parseMavenPom(descriptorURL, descriptorURL, repository);
286   
287  32 return coreExtension;
288    } catch (Exception e) {
289  0 this.logger.warn("Failed to parse extension descriptor [{}]", descriptorURL, e);
290    }
291    } else {
292  0 this.logger.warn("Can't find resource file [{}] which contains distribution informations.",
293    descriptorPath);
294    }
295    }
296    } catch (IOException e) {
297  0 this.logger.error("Failed to parse environment META-INF/MANIFEST.MF file", e);
298    } finally {
299  32 IOUtils.closeQuietly(manifestString);
300    }
301    }
302   
303    //////////
304    // Could not find any valid descriptor
305   
306  55 this.logger.debug("No declared environmennt extension");
307   
308  55 return null;
309    }
310   
 
311  87 toggle private Collection<URL> getJARs()
312    {
313    // ClasspathHelper.forClassLoader() get even the JARs that are made not reachable by the application server
314    // So the trick is to get all resources in which we can access a META-INF folder
315  87 Collection<URL> urls = ClasspathHelper.forPackage("META-INF");
316   
317  87 Collection<URL> jarURLs = new ArrayList<>(urls.size());
318   
319  87 for (URL url : urls) {
320  24667 try {
321  24667 jarURLs.add(PathUtils.getExtensionURL(url));
322    } catch (IOException e) {
323  0 this.logger.error("Failed to convert to extension URL", e);
324    }
325    }
326   
327  87 return jarURLs;
328    }
329   
 
330  22645 toggle private void addCoreExtension(Map<String, DefaultCoreExtension> extensions, DefaultCoreExtension coreExtension)
331    {
332  22645 DefaultCoreExtension existingCoreExtension = extensions.get(coreExtension.getId().getId());
333   
334  22645 if (existingCoreExtension == null) {
335  22100 extensions.put(coreExtension.getId().getId(), coreExtension);
336    } else {
337  545 Version existingVersion = existingCoreExtension.getId().getVersion();
338  545 Version version = coreExtension.getId().getVersion();
339   
340  545 int comparizon = version.compareTo(existingVersion);
341   
342    // Ignore collision between same versions
343  545 if (comparizon != 0) {
344  319 this.logger.warn("Collision between core extension [{} ({})] and [{} ({})]", coreExtension.getId(),
345    coreExtension.getDescriptorURL(), existingCoreExtension.getId(),
346    existingCoreExtension.getDescriptorURL());
347   
348  319 if (comparizon > 0) {
349  107 extensions.put(coreExtension.getId().getId(), coreExtension);
350   
351  107 this.logger.warn("[{} ({})] is selected", coreExtension.getId(), coreExtension.getDescriptorURL());
352    } else {
353  212 this.logger.warn("[{} ({})] is selected", existingCoreExtension.getId(),
354    existingCoreExtension.getDescriptorURL());
355    }
356    }
357    }
358    }
359   
 
360  24667 toggle private DefaultCoreExtension loadCoreExtensionFromXED(URL jarURL, DefaultCoreExtensionRepository repository)
361    {
362  24667 String jarString = jarURL.toExternalForm();
363   
364  24667 int extIndex = jarString.lastIndexOf('.');
365  24667 if (extIndex > 0) {
366    // Find XED file URL
367  24581 URL xedURL;
368  24581 try {
369  24581 xedURL = new URL(jarString.substring(0, extIndex) + ".xed");
370    } catch (MalformedURLException e) {
371    // Cannot really happen
372  0 return null;
373    }
374   
375    // Load XED stream
376  24581 InputStream xedStream;
377  24581 try {
378  24581 xedStream = xedURL.openStream();
379    } catch (IOException e) {
380    // We assume it means the xed does not exist so we just ignore it
381  13403 this.logger.debug("Failed to load [{}]", xedURL, e);
382  13403 return null;
383    }
384   
385    // Load XED file
386  11178 try {
387  11178 DefaultCoreExtension coreExtension =
388    this.parser.loadCoreExtensionDescriptor(repository, jarURL, xedStream);
389  11178 coreExtension.setDescriptorURL(xedURL);
390  11178 return coreExtension;
391    } catch (Exception e) {
392  0 this.logger.error("Failed to load [{}]", xedURL, e);
393    } finally {
394  11178 IOUtils.closeQuietly(xedStream);
395    }
396    }
397   
398  86 return null;
399    }
400   
 
401  87 toggle private void loadExtensionsFromClassloaders(Map<String, DefaultCoreExtension> extensions,
402    DefaultCoreExtensionRepository repository)
403    {
404    ////////////////////
405    // Get all jar files
406   
407  87 Collection<URL> jars = getJARs();
408   
409    ////////////////////
410    // Try to find associated xed files
411   
412  87 fromXED(extensions, jars, repository);
413   
414    ////////////////////
415    // Try to find associated Maven files
416   
417  87 fromMaven(extensions, jars, repository);
418   
419    ////////////////////
420    // Work some magic to guess the rest of the jar files
421   
422  87 guess(extensions, repository, jars);
423    }
424   
 
425  87 toggle private void fromXED(Map<String, DefaultCoreExtension> extensions, Collection<URL> jars,
426    DefaultCoreExtensionRepository repository)
427    {
428  24754 for (Iterator<URL> it = jars.iterator(); it.hasNext();) {
429  24667 URL jarURL = it.next();
430   
431  24667 DefaultCoreExtension coreExtension = loadCoreExtensionFromXED(jarURL, repository);
432   
433  24667 if (coreExtension != null) {
434    // Add the core extension
435  11178 addCoreExtension(extensions, coreExtension);
436   
437    // Remove the jar from the list
438  11178 it.remove();
439    }
440    }
441    }
442   
 
443  87 toggle private void fromMaven(Map<String, DefaultCoreExtension> extensions, Collection<URL> jars,
444    DefaultCoreExtensionRepository repository)
445    {
446  13576 for (Iterator<URL> it = jars.iterator(); it.hasNext();) {
447  13489 URL jar = it.next();
448   
449  13489 if (fromMaven(extensions, jar, repository)) {
450    // Remove the jar from the list
451  10306 it.remove();
452    }
453    }
454    }
455   
 
456  13489 toggle private boolean fromMaven(Map<String, DefaultCoreExtension> extensions, URL jarURL,
457    DefaultCoreExtensionRepository repository)
458    {
459  13489 ConfigurationBuilder configurationBuilder = new ConfigurationBuilder();
460  13489 configurationBuilder.setScanners(new ResourcesScanner());
461  13489 configurationBuilder.setUrls(jarURL);
462  13489 configurationBuilder.filterInputsBy(new FilterBuilder.Include(FilterBuilder.prefix(MavenUtils.MAVENPACKAGE)));
463   
464  13489 Reflections reflections = new Reflections(configurationBuilder);
465   
466    // We can get several pom.xml because the jar might embed several extensions
467  13489 Set<String> descriptors = reflections.getResources(Predicates.equalTo("pom.xml"));
468   
469  13489 boolean found = false;
470  13489 for (String descriptor : descriptors) {
471  11467 String path = jarURL.toExternalForm();
472   
473    // Create descriptor URL
474  11467 URL descriptorURL;
475  11467 try {
476  11467 if (path.endsWith("/")) {
477    // It's a folder
478  0 descriptorURL = new URL(path + descriptor);
479    } else {
480    // Probably a jar
481  11467 descriptorURL = new URL("jar:" + jarURL.toExternalForm() + "!/" + descriptor);
482    }
483    } catch (MalformedURLException e) {
484    // Not supposed to happen (would mean there is a bug in Reflections)
485  0 this.logger.error("Failed to access resource [{}] from jar [{}]", descriptor, jarURL);
486  0 continue;
487    }
488   
489  11467 try {
490    // Load Extension from descriptor
491  11467 DefaultCoreExtension coreExtension = getCoreExension(jarURL, descriptorURL, repository);
492   
493    // Add the core extension
494  11467 addCoreExtension(extensions, coreExtension);
495   
496  11467 found = true;
497    } catch (Exception e) {
498  0 this.logger.warn("Failed to parse extension descriptor [{}] ([{}])", descriptorURL, descriptor, e);
499    }
500    }
501   
502  13489 return found;
503    }
504   
 
505  87 toggle private void guess(Map<String, DefaultCoreExtension> extensions, DefaultCoreExtensionRepository repository,
506    Collection<URL> jars)
507    {
508  87 Set<ExtensionDependency> dependencies = new HashSet<>();
509   
510  87 for (DefaultCoreExtension coreExtension : extensions.values()) {
511  22100 for (ExtensionDependency dependency : coreExtension.getDependencies()) {
512  65620 if (!extensions.containsKey(dependency.getId())) {
513  8021 dependencies.add(dependency);
514    }
515    }
516    }
517   
518    // Normalize and guess
519   
520  87 Map<String, Object[]> fileNames = new HashMap<>();
521  87 Map<String, Object[]> guessedArtefacts = new HashMap<>();
522   
523  3270 for (Iterator<URL> it = jars.iterator(); it.hasNext();) {
524  3183 URL jarURL = it.next();
525   
526  3183 try {
527  3183 String path = jarURL.getPath();
528  3183 String filename = path.substring(path.lastIndexOf('/') + 1);
529  3183 String type = null;
530   
531  3183 int extIndex = filename.lastIndexOf('.');
532  3183 if (extIndex != -1) {
533  3097 type = filename.substring(extIndex + 1);
534  3097 filename = filename.substring(0, extIndex);
535    }
536   
537  3183 int index;
538  3183 if (!filename.endsWith(MavenUtils.SNAPSHOTSUFFIX)) {
539  3127 index = filename.lastIndexOf('-');
540    } else {
541  56 index = filename.lastIndexOf('-', filename.length() - MavenUtils.SNAPSHOTSUFFIX.length());
542    }
543   
544  3183 if (index != -1) {
545  3070 fileNames.put(filename, new Object[] { jarURL });
546   
547  3070 String artefactname = filename.substring(0, index);
548  3070 String version = filename.substring(index + 1);
549   
550  3070 guessedArtefacts.put(artefactname, new Object[] { version, jarURL, type });
551    }
552    } catch (Exception e) {
553  0 this.logger.warn("Failed to parse resource name [{}]", jarURL, e);
554    }
555    }
556   
557    // Try to resolve version no easy to find from the pom.xml
558  87 try {
559  87 for (DefaultCoreExtension coreExtension : extensions.values()) {
560  22100 String artifactId = getArtifactId(coreExtension);
561   
562  22100 Object[] artefact = guessedArtefacts.get(artifactId);
563   
564  22100 if (artefact != null) {
565  0 if (coreExtension.getId().getVersion().getValue().charAt(0) == '$') {
566  0 coreExtension.setId(new ExtensionId(coreExtension.getId().getId(), (String) artefact[0]));
567  0 coreExtension.setGuessed(true);
568    }
569   
570  0 if (coreExtension.getType().charAt(0) == '$') {
571  0 coreExtension.setType((String) artefact[2]);
572  0 coreExtension.setGuessed(true);
573    }
574    }
575    }
576   
577    // Add dependencies that does not provide proper pom.xml resource and can't be found in the classpath
578  87 for (ExtensionDependency extensionDependency : dependencies) {
579  6975 Dependency dependency;
580   
581  6975 if (extensionDependency instanceof MavenExtensionDependency) {
582  5436 dependency = ((MavenExtensionDependency) extensionDependency).getMavenDependency();
583    } else {
584  1539 dependency = toDependency(extensionDependency.getId(),
585    extensionDependency.getVersionConstraint().getValue(), null);
586    }
587   
588  6975 String dependencyId = dependency.getGroupId() + ':' + dependency.getArtifactId();
589   
590  6975 DefaultCoreExtension coreExtension = extensions.get(dependencyId);
591  6975 if (coreExtension == null) {
592  6055 String dependencyFileName = dependency.getArtifactId() + '-' + dependency.getVersion();
593  6055 if (dependency.getClassifier() != null) {
594  95 dependencyFileName += '-' + dependency.getClassifier();
595  95 dependencyId += ':' + dependency.getClassifier();
596    }
597   
598  6055 Object[] filenameArtifact = fileNames.get(dependencyFileName);
599  6055 Object[] guessedArtefact = guessedArtefacts.get(dependency.getArtifactId());
600   
601  6055 if (filenameArtifact != null) {
602  565 coreExtension = new DefaultCoreExtension(repository, (URL) filenameArtifact[0],
603    new ExtensionId(dependencyId, dependency.getVersion()),
604    MavenUtils.packagingToType(dependency.getType()));
605  565 coreExtension.setGuessed(true);
606  5490 } else if (guessedArtefact != null) {
607  1435 coreExtension = new DefaultCoreExtension(repository, (URL) guessedArtefact[1],
608    new ExtensionId(dependencyId, (String) guessedArtefact[0]),
609    MavenUtils.packagingToType(dependency.getType()));
610  1435 coreExtension.setGuessed(true);
611    }
612   
613  6055 if (coreExtension != null) {
614  2000 extensions.put(dependencyId, coreExtension);
615    }
616    }
617    }
618    } catch (Exception e) {
619  0 this.logger.warn("Failed to guess extra information about some extensions", e);
620    }
621    }
622    }