Mule stream design to handle large files from a remote location
I have a thread with the following steps:
1) Select file from SFTP
sever source
2) Copy it to local storage
3) Process file using copy in local storage
4) Put the processed file (to be converted) to the target server SFTP
5) Move the file present in the source SFTP
to a different folder on the original server SFTP
(I couldn't find a way to do this, and hence I copy from the temporary location back to the SFTP
processed folder)
It looks like a standard workflow, however I couldn't find any advice on how to specifically implement this in Mule.
My current implementation is described below:
<file:connector name="tempFile" workDirectory="${temp.file.location}/work"
workFileNamePattern="#[message.inboundProperties.originalFilename]"
autoDelete="true" streaming="false" validateConnections="true"
doc:name="File" />
<sftp:connector name="InputSFTP" validateConnections="true" keepFileOnError="true" doc:name="SFTP" >
<reconnect frequency="${reconnectfrequency}" count="5"/>
</sftp:connector>
<sftp:connector name="DestinationSFTP" validateConnections="true" pollingFrequency="30000" doc:name="SFTP">
<reconnect frequency="${reconnectfrequency}" count="5"/>
</sftp:connector>
<smtp:gmail-connector name="Gmail" contentType="text/plain" validateConnections="true" doc:name="Gmail"/>
<flow name="DownloadFTPFileIntoLocalFlow" processingStrategy="synchronous" tracking:enable-default-events="true">
<sftp:inbound-endpoint connector-ref="InputSFTP" host="${source.host}" port="22" path="${source.path}" user="${source.username}"
password="${source.password}" responseTimeout="90000" pollingFrequency="120000" sizeCheckWaitTime="1000" doc:name="InputSFTP" autoDelete="true">
<file:filename-regex-filter pattern="[Z].*\.csv" caseSensitive="false" />
</sftp:inbound-endpoint>
<file:outbound-endpoint path="${temp.file.location}" responseTimeout="10000" doc:name="Templocation" outputPattern="#[message.inboundProperties.originalFilename]" connector-ref="tempFile" />
<exception-strategy ref="Default_Exception_Strategy" doc:name="Reference Exception Strategy"/>
</flow>
<flow name="ProcessCSVFlow" processingStrategy="synchronous" tracking:enable-default-events="true">
<file:inbound-endpoint path="${temp.file.location}" connector-ref="tempFile" pollingFrequency="180000" fileAge="10000" responseTimeout="10000" doc:name="TempFileLocation"/>
<transformer ref="enrichWithHeaderAndEndOfFileTransformer" doc:name="headerAndEOFEnricher" />
<set-variable variableName="outputfilename" value="#['Mercury'+server.dateTime.year+server.dateTime.month+server.dateTime.dayOfMonth+server.dateTime.hours +server.dateTime.minutes+server.dateTime.seconds+'.csv']" doc:name="outputfilename"/>
<sftp:outbound-endpoint exchange-pattern="one-way" connector-ref="DestinationSFTP" host="${destination.host}" port="22" responseTimeout="10000" doc:name="DestinationSFTP"
outputPattern="#[outputfilename]" path="${destination.path}" user="${destination.username}" password="${destination.password}"/>
<gzip-compress-transformer/>
<sftp:outbound-endpoint exchange-pattern="one-way" connector-ref="InputSFTP" host="${source.host}" port="22" responseTimeout="10000" doc:name="SourceArchiveSFTP"
outputPattern="#[outputfilename].gzip" path="Archive" user="${source.username}" password="${source.password}"/>
<set-payload value="Hello world" doc:name="Set Payload"/>
<smtp:outbound-endpoint host="${smtp.host}" port="${smtp.port}" user="${smtp.from.address}" password="${smtp.from.password}"
to="${smtp.to.address}" from="${smtp.from.address}" subject="${mail.success.subject}" responseTimeout="10000"
doc:name="SuccessEmail" connector-ref="Gmail"/>
<logger message="Process completed successfully" level="INFO" doc:name="Logger"/>
<tracking:transaction id="#[server.dateTime]"/>
<exception-strategy ref="Default_Exception_Strategy" doc:name="Reference Exception Strategy"/>
</flow>
<catch-exception-strategy name="Default_Exception_Strategy">
<logger message="Exception has occured Payload is #[payload] and Message is #[message]" level="ERROR" doc:name="Logger"/>
<!-- <smtp:outbound-endpoint host="localhost" responseTimeout="10000" doc:name="Failure Email"/> -->
</catch-exception-strategy>
source to share