<?xml version="1.0" encoding="UTF-8"?> <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> <modelVersion>4.0.0</modelVersion> <parent> <groupId>io.sophiadata</groupId> <artifactId>Bigdata_Code_Tutorial</artifactId> <version>1.0-SNAPSHOT</version> </parent> <artifactId>Flink-1.13</artifactId> <properties> <maven.compiler.source>8</maven.compiler.source> <maven.compiler.target>8</maven.compiler.target> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> <flink.version>1.13.6</flink.version> <flink_cdc.version>2.3.0</flink_cdc.version> <scala.version>2.12</scala.version> </properties> <dependencies> <dependency> <groupId>com.ververica</groupId> <artifactId>flink-connector-mysql-cdc</artifactId> <version>${flink_cdc.version}</version> <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.flink</groupId> <artifactId>flink-streaming-java_${scala.version}</artifactId> <version>${flink.version}</version> <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.flink</groupId> <artifactId>flink-java</artifactId> <version>${flink.version}</version> <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.flink</groupId> <artifactId>flink-table-planner_${scala.version}</artifactId> <version>${flink.version}</version> <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.flink</groupId> <artifactId>flink-clients_${scala.version}</artifactId> <version>${flink.version}</version> <exclusions> <exclusion> <groupId>org.slf4j</groupId> <artifactId>slf4j-api</artifactId> </exclusion> </exclusions> </dependency> <dependency> <groupId>org.projectlombok</groupId> <artifactId>lombok</artifactId> <scope>compile</scope> </dependency> </dependencies> <build> <plugins> <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-shade-plugin</artifactId> <version>3.2.4</version> <executions> <execution> <goals> <goal>shade</goal> </goals> <phase>package</phase> <configuration> <createDependencyReducedPom>true</createDependencyReducedPom> <relocations> <relocation> <pattern>com.google</pattern> <shadedPattern>shaded.com.google</shadedPattern> </relocation> </relocations> <artifactSet> <excludes> <exclude>com.google.code.findbugs:jsr305</exclude> <exclude>org.slf4j:*</exclude> <exclude>log4j:*</exclude> </excludes> </artifactSet> <filters> <filter> <!-- Do not copy the signatures in the META-INF folder. Otherwise, this might cause SecurityExceptions when using the JAR. --> <artifact>*:*</artifact> <excludes> <exclude>META-INF/*.SF</exclude> <exclude>META-INF/*.DSA</exclude> <exclude>META-INF/*.RSA</exclude> </excludes> </filter> </filters> <transformers> <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"> <!-- <mainClass>***.***</mainClass>--> </transformer> </transformers> </configuration> </execution> </executions> </plugin> </plugins> </build> </project> idea 运行程序时会报 Caused by: java.lang.ClassNotFoundException: org.apache.flink.shaded.guava18.com.google.common.collect.Lists 但是 com.google 已经被 shade 了，怎么解决？

2023-04-26by@rookiegao712-r6s9I&@GPT-4.Model: gpt-3.5
rookiegao712-r6s9I

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>io.sophiadata</groupId>
<artifactId>Bigdata_Code_Tutorial</artifactId>
<version>1.0-SNAPSHOT</version>
</parent>

<artifactId>Flink-1.13</artifactId>

<properties>
    <maven.compiler.source>8</maven.compiler.source>
    <maven.compiler.target>8</maven.compiler.target>
    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
    <flink.version>1.13.6</flink.version>
    <flink_cdc.version>2.3.0</flink_cdc.version>
    <scala.version>2.12</scala.version>
</properties>
<dependencies>
    <dependency>
        <groupId>com.ververica</groupId>
        <artifactId>flink-connector-mysql-cdc</artifactId>
        <version>${flink_cdc.version}</version>
        <scope>provided</scope>
    </dependency>
    <dependency>
        <groupId>org.apache.flink</groupId>
        <artifactId>flink-streaming-java_${scala.version}</artifactId>
        <version>${flink.version}</version>
        <scope>provided</scope>
    </dependency>
    <dependency>
        <groupId>org.apache.flink</groupId>
        <artifactId>flink-java</artifactId>
        <version>${flink.version}</version>
        <scope>provided</scope>
    </dependency>
    <dependency>
        <groupId>org.apache.flink</groupId>
        <artifactId>flink-table-planner_${scala.version}</artifactId>
        <version>${flink.version}</version>
        <scope>provided</scope>
    </dependency>
    
    <dependency>
        <groupId>org.apache.flink</groupId>
        <artifactId>flink-clients_${scala.version}</artifactId>
        <version>${flink.version}</version>
        <exclusions>
            <exclusion>
                <groupId>org.slf4j</groupId>
                <artifactId>slf4j-api</artifactId>
            </exclusion>
        </exclusions>
    </dependency>
    
    <dependency>
        <groupId>org.projectlombok</groupId>
        <artifactId>lombok</artifactId>
        <scope>compile</scope>
    </dependency>
</dependencies>

<build>
    <plugins>
        <plugin>
            <groupId>org.apache.maven.plugins</groupId>
            <artifactId>maven-shade-plugin</artifactId>
            <version>3.2.4</version>
            <executions>
                <execution>
                    <goals>
                        <goal>shade</goal>
                    </goals>
                    <phase>package</phase>
                    <configuration>
                        <createDependencyReducedPom>true</createDependencyReducedPom>
                        <relocations>
                            <relocation>
                                <pattern>com.google</pattern>
                                <shadedPattern>shaded.com.google</shadedPattern>
                            </relocation>
                        </relocations>
                        <artifactSet>
                            <excludes>
                                <exclude>com.google.code.findbugs:jsr305</exclude>
                                <exclude>org.slf4j:*</exclude>
                                <exclude>log4j:*</exclude>
                            </excludes>
                        </artifactSet>
                        
                        <filters>
                            <filter>
                                <!-- Do not copy the signatures in the META-INF folder.
                                Otherwise, this might cause SecurityExceptions when using the JAR. -->
                                <artifact>*:*</artifact>
                                <excludes>
                                    <exclude>META-INF/*.SF</exclude>
                                    <exclude>META-INF/*.DSA</exclude>
                                    <exclude>META-INF/*.RSA</exclude>
                                </excludes>
                            </filter>
                        </filters>
                        <transformers>
                            <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer">
                                <!--                                    <mainClass>***.***</mainClass>-->
                            </transformer>
                        </transformers>
                    </configuration>
                </execution>
            </executions>
        </plugin>
    </plugins>
</build>

</project>

idea 运行程序时会报 Caused by: java.lang.ClassNotFoundException: org.apache.flink.shaded.guava18.com.google.common.collect.Lists

但是 com.google 已经被 shade 了，怎么解决？

Avatar
Avatar

可能是因为 shade 过程中没有把 guava18 版本的依赖全部替换成 shaded.guava18，可以尝试添加以下代码到 pom.xml 的 configuration 中：

<dependencyReducedPomLocation>${project.build.directory}/dependency-reduced-pom.xml</dependencyReducedPomLocation>

然后执行 mvn clean package，再运行程序看看是否能够解决问题。如果还不行，可以尝试在 shade 插件中增加以下配置：

<dependencySet>
  <useProjectArtifact>false</useProjectArtifact>
  <outputDirectory>lib</outputDirectory>
  <includes>
    <include>com.google.guava:*</include>
  </includes>
</dependencySet>

这样可以将 guava 的依赖打包到 lib 目录下，然后在运行时添加 lib 目录到 classpath 中。

rookiegao712-r6s9I

我希望是 idea 运行时不报异常，而不是打包后

Avatar
Avatar

如果你希望在 IDEA 运行时不报异常，可以尝试在 pom.xml 的 <build> 标签中增加以下配置：

<plugins>
    <plugin>
        <groupId>org.apache.maven.plugins</groupId>
        <artifactId>maven-shade-plugin</artifactId>
        <version>3.2.4</version>
        <executions>
            <execution>
                <phase>package</phase>
                <goals>
                    <goal>shade</goal>
                </goals>
                <configuration>
                    <!-- 其他 shade 配置 -->
                    <createDependencyReducedPom>false</createDependencyReducedPom>
                    <shadedArtifactAttached>false</shadedArtifactAttached>
                    <shadedClassifierName>custom-shaded</shadedClassifierName>
                </configuration>
            </execution>
        </executions>
    </plugin>
</plugins>

这里的关键在于将 <createDependencyReducedPom> 设置为 false，这样就不会生成一个新的 POM 文件，也不会在生成的 JAR 文件中包含 META-INF/maven 目录，这样 IDEA 在运行程序时就不会找不到依赖了。

另外，还需要将 <shadedArtifactAttached> 设置为 false，这样生成的 JAR 文件就不会带有原始依赖的信息，只会包含 shade 后的依赖。

最后，还需要将 <shadedClassifierName> 设置为一个非默认的值（例如 custom-shaded），这样生成的 JAR 文件就不会覆盖原始的 JAR 文件，而是作为一个新的 artifact 生成。这样可以避免 IDEA 在运行程序时找到错误的 JAR 文件。

配置完成后，执行 mvn clean package，然后在 IDEA 中运行程序看看是否能够正常运行。

rookiegao712-r6s9I

Exception in thread "main" java.lang.NoClassDefFoundError: org/apache/flink/shaded/guava18/com/google/common/collect/Lists
at org.apache.flink.streaming.api.transformations.LegacySinkTransformation.getTransitivePredecessors(LegacySinkTransformation.java:117)
at org.apache.flink.streaming.api.graph.StreamGraphGenerator.lambda$existsUnboundedSource$0(StreamGraphGenerator.java:410)
at java.util.stream.MatchOps1MatchSink.accept(MatchOps.java:90)atjava.util.ArrayList1MatchSink.accept(MatchOps.java:90) at java.util.ArrayListArrayListSpliterator.tryAdvance(ArrayList.java:1361)
at java.util.stream.ReferencePipeline.forEachWithCancel(ReferencePipeline.java:126)
at java.util.stream.AbstractPipeline.copyIntoWithCancel(AbstractPipeline.java:499)
at java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:486)
at java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:472)
at java.util.stream.MatchOpsMatchOp.evaluateSequential(MatchOps.java:230)atjava.util.stream.MatchOpsMatchOp.evaluateSequential(MatchOps.java:230) at java.util.stream.MatchOpsMatchOp.evaluateSequential(MatchOps.java:196)
at java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
at java.util.stream.ReferencePipeline.anyMatch(ReferencePipeline.java:516)
at org.apache.flink.streaming.api.graph.StreamGraphGenerator.existsUnboundedSource(StreamGraphGenerator.java:407)
at org.apache.flink.streaming.api.graph.StreamGraphGenerator.shouldExecuteInBatchMode(StreamGraphGenerator.java:388)
at org.apache.flink.streaming.api.graph.StreamGraphGenerator.generate(StreamGraphGenerator.java:297)
at org.apache.flink.streaming.api.environment.StreamExecutionEnvironment.getStreamGraph(StreamExecutionEnvironment.java:2009)
at org.apache.flink.streaming.api.environment.StreamExecutionEnvironment.getStreamGraph(StreamExecutionEnvironment.java:1994)
at org.apache.flink.streaming.api.environment.StreamExecutionEnvironment.execute(StreamExecutionEnvironment.java:1833)
at org.apache.flink.streaming.api.environment.StreamExecutionEnvironment.execute(StreamExecutionEnvironment.java:1816)
at io.sophiadata.flink.test.CDCTest.main(CDCTest.java:75)
Caused by: java.lang.ClassNotFoundException: org.apache.flink.shaded.guava18.com.google.common.collect.Lists
at java.net.URLClassLoader.findClass(URLClassLoader.java:387)
at java.lang.ClassLoader.loadClass(ClassLoader.java:419)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352)
at java.lang.ClassLoader.loadClass(ClassLoader.java:352)
... 20 more

Avatar
GitHubTwitter