1*eeb7e5b3SAdam Hornáček<?xml version="1.0" encoding="UTF-8"?> 2*eeb7e5b3SAdam Hornáček<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" 3*eeb7e5b3SAdam Hornáček "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> 4*eeb7e5b3SAdam Hornáček<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en" 5*eeb7e5b3SAdam Hornáček class="xref"> 6*eeb7e5b3SAdam Hornáček<head> 7*eeb7e5b3SAdam Hornáček<title>sampleFile - OpenGrok cross reference for /sampleFile</title></head><body> 8*eeb7e5b3SAdam Hornáček<script type="text/javascript">/* <![CDATA[ */ 9*eeb7e5b3SAdam Hornáčekfunction get_sym_list(){return [["Package","xp",[["org.apache.spark.broadcast",18]]],["Method","xmt",[["blockifyObject",270],["unBlockifyObject",287],["unary_",318],["unpersist",308]]]];} /* ]]> */</script><a class="l" name="1" href="#1">1</a><span class="c">/*</span> 10*eeb7e5b3SAdam Hornáček<a class="l" name="2" href="#2">2</a><span class="c"> * Licensed to the Apache Software Foundation (ASF) under one or more</span> 11*eeb7e5b3SAdam Hornáček<a class="l" name="3" href="#3">3</a><span class="c"> * contributor license agreements. See the NOTICE file distributed with</span> 12*eeb7e5b3SAdam Hornáček<a class="l" name="4" href="#4">4</a><span class="c"> * this work for additional information regarding copyright ownership.</span> 13*eeb7e5b3SAdam Hornáček<a class="l" name="5" href="#5">5</a><span class="c"> * The ASF licenses this file to You under the Apache License, Version 2.0</span> 14*eeb7e5b3SAdam Hornáček<a class="l" name="6" href="#6">6</a><span class="c"> * (the "License"); you may not use this file except in compliance with</span> 15*eeb7e5b3SAdam Hornáček<a class="l" name="7" href="#7">7</a><span class="c"> * the License. You may obtain a copy of the License at</span> 16*eeb7e5b3SAdam Hornáček<a class="l" name="8" href="#8">8</a><span class="c"> *</span> 17*eeb7e5b3SAdam Hornáček<a class="l" name="9" href="#9">9</a><span class="c"> * <a href="http://www.apache.org/licenses/LICENSE-2.0">http://www.apache.org/licenses/LICENSE-2.0</a></span> 18*eeb7e5b3SAdam Hornáček<a class="hl" name="10" href="#10">10</a><span class="c"> *</span> 19*eeb7e5b3SAdam Hornáček<a class="l" name="11" href="#11">11</a><span class="c"> * Unless required by applicable law or agreed to in writing, software</span> 20*eeb7e5b3SAdam Hornáček<a class="l" name="12" href="#12">12</a><span class="c"> * distributed under the License is distributed on an "AS IS" BASIS,</span> 21*eeb7e5b3SAdam Hornáček<a class="l" name="13" href="#13">13</a><span class="c"> * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span> 22*eeb7e5b3SAdam Hornáček<a class="l" name="14" href="#14">14</a><span class="c"> * See the License for the specific language governing permissions and</span> 23*eeb7e5b3SAdam Hornáček<a class="l" name="15" href="#15">15</a><span class="c"> * limitations under the License.</span> 24*eeb7e5b3SAdam Hornáček<a class="l" name="16" href="#16">16</a><span class="c"> */</span> 25*eeb7e5b3SAdam Hornáček<a class="l" name="17" href="#17">17</a> 26*eeb7e5b3SAdam Hornáček<a class="l" name="18" href="#18">18</a><b>package</b> <a href="/source/s?defs=org" class="intelliWindow-symbol" data-definition-place="undefined-in-file">org</a>.<a href="/source/s?defs=apache" class="intelliWindow-symbol" data-definition-place="undefined-in-file">apache</a>.<a href="/source/s?defs=spark" class="intelliWindow-symbol" data-definition-place="undefined-in-file">spark</a>.<a href="/source/s?defs=broadcast" class="intelliWindow-symbol" data-definition-place="undefined-in-file">broadcast</a> 27*eeb7e5b3SAdam Hornáček<a class="l" name="19" href="#19">19</a> 28*eeb7e5b3SAdam Hornáček<a class="hl" name="20" href="#20">20</a><b>import</b> <a href="/source/s?defs=java" class="intelliWindow-symbol" data-definition-place="undefined-in-file">java</a>.<a href="/source/s?defs=io" class="intelliWindow-symbol" data-definition-place="undefined-in-file">io</a>.<b>_</b> 29*eeb7e5b3SAdam Hornáček<a class="l" name="21" href="#21">21</a><b>import</b> <a href="/source/s?defs=java" class="intelliWindow-symbol" data-definition-place="undefined-in-file">java</a>.<a href="/source/s?defs=nio" class="intelliWindow-symbol" data-definition-place="undefined-in-file">nio</a>.<a href="/source/s?defs=ByteBuffer" class="intelliWindow-symbol" data-definition-place="undefined-in-file">ByteBuffer</a> 30*eeb7e5b3SAdam Hornáček<a class="l" name="22" href="#22">22</a><b>import</b> <a href="/source/s?defs=java" class="intelliWindow-symbol" data-definition-place="undefined-in-file">java</a>.<a href="/source/s?defs=util" class="intelliWindow-symbol" data-definition-place="undefined-in-file">util</a>.<a href="/source/s?defs=zip" class="intelliWindow-symbol" data-definition-place="undefined-in-file">zip</a>.<a href="/source/s?defs=Adler32" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Adler32</a> 31*eeb7e5b3SAdam Hornáček<a class="l" name="23" href="#23">23</a> 32*eeb7e5b3SAdam Hornáček<a class="l" name="24" href="#24">24</a><b>import</b> <a href="/source/s?defs=scala" class="intelliWindow-symbol" data-definition-place="undefined-in-file">scala</a>.<a href="/source/s?defs=collection" class="intelliWindow-symbol" data-definition-place="undefined-in-file">collection</a>.<a href="/source/s?defs=JavaConverters" class="intelliWindow-symbol" data-definition-place="undefined-in-file">JavaConverters</a>.<b>_</b> 33*eeb7e5b3SAdam Hornáček<a class="l" name="25" href="#25">25</a><b>import</b> <a href="/source/s?defs=scala" class="intelliWindow-symbol" data-definition-place="undefined-in-file">scala</a>.<a href="/source/s?defs=reflect" class="intelliWindow-symbol" data-definition-place="undefined-in-file">reflect</a>.<a href="/source/s?defs=ClassTag" class="intelliWindow-symbol" data-definition-place="undefined-in-file">ClassTag</a> 34*eeb7e5b3SAdam Hornáček<a class="l" name="26" href="#26">26</a><b>import</b> <a href="/source/s?defs=scala" class="intelliWindow-symbol" data-definition-place="undefined-in-file">scala</a>.<a href="/source/s?defs=util" class="intelliWindow-symbol" data-definition-place="undefined-in-file">util</a>.<a href="/source/s?defs=Random" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Random</a> 35*eeb7e5b3SAdam Hornáček<a class="l" name="27" href="#27">27</a> 36*eeb7e5b3SAdam Hornáček<a class="l" name="28" href="#28">28</a><b>import</b> <a href="/source/s?defs=org" class="intelliWindow-symbol" data-definition-place="undefined-in-file">org</a>.<a href="/source/s?defs=apache" class="intelliWindow-symbol" data-definition-place="undefined-in-file">apache</a>.<a href="/source/s?defs=spark" class="intelliWindow-symbol" data-definition-place="undefined-in-file">spark</a>.<b>_</b> 37*eeb7e5b3SAdam Hornáček<a class="l" name="29" href="#29">29</a><b>import</b> <a href="/source/s?defs=org" class="intelliWindow-symbol" data-definition-place="undefined-in-file">org</a>.<a href="/source/s?defs=apache" class="intelliWindow-symbol" data-definition-place="undefined-in-file">apache</a>.<a href="/source/s?defs=spark" class="intelliWindow-symbol" data-definition-place="undefined-in-file">spark</a>.<a href="/source/s?defs=internal" class="intelliWindow-symbol" data-definition-place="undefined-in-file">internal</a>.<a href="/source/s?defs=Logging" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Logging</a> 38*eeb7e5b3SAdam Hornáček<a class="hl" name="30" href="#30">30</a><b>import</b> <a href="/source/s?defs=org" class="intelliWindow-symbol" data-definition-place="undefined-in-file">org</a>.<a href="/source/s?defs=apache" class="intelliWindow-symbol" data-definition-place="undefined-in-file">apache</a>.<a href="/source/s?defs=spark" class="intelliWindow-symbol" data-definition-place="undefined-in-file">spark</a>.<a href="/source/s?defs=io" class="intelliWindow-symbol" data-definition-place="undefined-in-file">io</a>.<a href="/source/s?defs=CompressionCodec" class="intelliWindow-symbol" data-definition-place="undefined-in-file">CompressionCodec</a> 39*eeb7e5b3SAdam Hornáček<a class="l" name="31" href="#31">31</a><b>import</b> <a href="/source/s?defs=org" class="intelliWindow-symbol" data-definition-place="undefined-in-file">org</a>.<a href="/source/s?defs=apache" class="intelliWindow-symbol" data-definition-place="undefined-in-file">apache</a>.<a href="/source/s?defs=spark" class="intelliWindow-symbol" data-definition-place="undefined-in-file">spark</a>.<a href="/source/s?defs=serializer" class="intelliWindow-symbol" data-definition-place="undefined-in-file">serializer</a>.<a href="/source/s?defs=Serializer" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Serializer</a> 40*eeb7e5b3SAdam Hornáček<a class="l" name="32" href="#32">32</a><b>import</b> <a href="/source/s?defs=org" class="intelliWindow-symbol" data-definition-place="undefined-in-file">org</a>.<a href="/source/s?defs=apache" class="intelliWindow-symbol" data-definition-place="undefined-in-file">apache</a>.<a href="/source/s?defs=spark" class="intelliWindow-symbol" data-definition-place="undefined-in-file">spark</a>.<a href="/source/s?defs=storage" class="intelliWindow-symbol" data-definition-place="undefined-in-file">storage</a>.<b>_</b> 41*eeb7e5b3SAdam Hornáček<a class="l" name="33" href="#33">33</a><b>import</b> <a href="/source/s?defs=org" class="intelliWindow-symbol" data-definition-place="undefined-in-file">org</a>.<a href="/source/s?defs=apache" class="intelliWindow-symbol" data-definition-place="undefined-in-file">apache</a>.<a href="/source/s?defs=spark" class="intelliWindow-symbol" data-definition-place="undefined-in-file">spark</a>.<a href="/source/s?defs=util" class="intelliWindow-symbol" data-definition-place="undefined-in-file">util</a>.<a href="/source/s?defs=Utils" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Utils</a> 42*eeb7e5b3SAdam Hornáček<a class="l" name="34" href="#34">34</a><b>import</b> <a href="/source/s?defs=org" class="intelliWindow-symbol" data-definition-place="undefined-in-file">org</a>.<a href="/source/s?defs=apache" class="intelliWindow-symbol" data-definition-place="undefined-in-file">apache</a>.<a href="/source/s?defs=spark" class="intelliWindow-symbol" data-definition-place="undefined-in-file">spark</a>.<a href="/source/s?defs=util" class="intelliWindow-symbol" data-definition-place="undefined-in-file">util</a>.<a href="/source/s?defs=io" class="intelliWindow-symbol" data-definition-place="undefined-in-file">io</a>.{<a href="/source/s?defs=ChunkedByteBuffer" class="intelliWindow-symbol" data-definition-place="undefined-in-file">ChunkedByteBuffer</a>, <a href="/source/s?defs=ChunkedByteBufferOutputStream" class="intelliWindow-symbol" data-definition-place="undefined-in-file">ChunkedByteBufferOutputStream</a>} 43*eeb7e5b3SAdam Hornáček<a class="l" name="35" href="#35">35</a> 44*eeb7e5b3SAdam Hornáček<a class="l" name="36" href="#36">36</a><span class="c">/**</span> 45*eeb7e5b3SAdam Hornáček<a class="l" name="37" href="#37">37</a><span class="c"> * A BitTorrent-like implementation of [[org.apache.spark.broadcast.Broadcast]].</span> 46*eeb7e5b3SAdam Hornáček<a class="l" name="38" href="#38">38</a><span class="c"> *</span> 47*eeb7e5b3SAdam Hornáček<a class="l" name="39" href="#39">39</a><span class="c"> * The mechanism is as follows:</span> 48*eeb7e5b3SAdam Hornáček<a class="hl" name="40" href="#40">40</a><span class="c"> *</span> 49*eeb7e5b3SAdam Hornáček<a class="l" name="41" href="#41">41</a><span class="c"> * The driver divides the serialized object into small chunks and</span> 50*eeb7e5b3SAdam Hornáček<a class="l" name="42" href="#42">42</a><span class="c"> * stores those chunks in the BlockManager of the driver.</span> 51*eeb7e5b3SAdam Hornáček<a class="l" name="43" href="#43">43</a><span class="c"> *</span> 52*eeb7e5b3SAdam Hornáček<a class="l" name="44" href="#44">44</a><span class="c"> * On each executor, the executor first attempts to fetch the object from its BlockManager. If</span> 53*eeb7e5b3SAdam Hornáček<a class="l" name="45" href="#45">45</a><span class="c"> * it does not exist, it then uses remote fetches to fetch the small chunks from the driver <a href="/source/s?path=and/">and</a>/<a href="/source/s?path=and/or">or</a></span> 54*eeb7e5b3SAdam Hornáček<a class="l" name="46" href="#46">46</a><span class="c"> * other executors if available. Once it gets the chunks, it puts the chunks in its own</span> 55*eeb7e5b3SAdam Hornáček<a class="l" name="47" href="#47">47</a><span class="c"> * BlockManager, ready for other executors to fetch from.</span> 56*eeb7e5b3SAdam Hornáček<a class="l" name="48" href="#48">48</a><span class="c"> *</span> 57*eeb7e5b3SAdam Hornáček<a class="l" name="49" href="#49">49</a><span class="c"> * This prevents the driver from being the bottleneck in sending out multiple copies of the</span> 58*eeb7e5b3SAdam Hornáček<a class="hl" name="50" href="#50">50</a><span class="c"> * broadcast data (one per executor).</span> 59*eeb7e5b3SAdam Hornáček<a class="l" name="51" href="#51">51</a><span class="c"> *</span> 60*eeb7e5b3SAdam Hornáček<a class="l" name="52" href="#52">52</a><span class="c"> * When initialized, TorrentBroadcast objects read <a href="/source/s?path=SparkEnv.get.conf">SparkEnv.get.conf</a>.</span> 61*eeb7e5b3SAdam Hornáček<a class="l" name="53" href="#53">53</a><span class="c"> *</span> 62*eeb7e5b3SAdam Hornáček<a class="l" name="54" href="#54">54</a><span class="c"> * <strong>@param</strong> <em>obj</em> object to broadcast</span> 63*eeb7e5b3SAdam Hornáček<a class="l" name="55" href="#55">55</a><span class="c"> * <strong>@param</strong> <em>id</em> A unique identifier for the broadcast variable.</span> 64*eeb7e5b3SAdam Hornáček<a class="l" name="56" href="#56">56</a><span class="c"> */</span> 65*eeb7e5b3SAdam Hornáček<a class="l" name="57" href="#57">57</a><b>private</b>[<a href="/source/s?defs=spark" class="intelliWindow-symbol" data-definition-place="undefined-in-file">spark</a>] <b>class</b> <a class="d intelliWindow-symbol" href="#TorrentBroadcast" data-definition-place="defined-in-file">TorrentBroadcast</a>[<a href="/source/s?defs=T" class="intelliWindow-symbol" data-definition-place="undefined-in-file">T</a>: <a href="/source/s?defs=ClassTag" class="intelliWindow-symbol" data-definition-place="undefined-in-file">ClassTag</a>](<a href="/source/s?defs=obj" class="intelliWindow-symbol" data-definition-place="undefined-in-file">obj</a>: <a href="/source/s?defs=T" class="intelliWindow-symbol" data-definition-place="undefined-in-file">T</a>, <a href="/source/s?defs=id" class="intelliWindow-symbol" data-definition-place="undefined-in-file">id</a>: <a href="/source/s?defs=Long" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Long</a>) 66*eeb7e5b3SAdam Hornáček<a class="l" name="58" href="#58">58</a> <b>extends</b> <a href="/source/s?defs=Broadcast" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Broadcast</a>[<a href="/source/s?defs=T" class="intelliWindow-symbol" data-definition-place="undefined-in-file">T</a>](<a href="/source/s?defs=id" class="intelliWindow-symbol" data-definition-place="undefined-in-file">id</a>) <b>with</b> <a href="/source/s?defs=Logging" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Logging</a> <b>with</b> <a href="/source/s?defs=Serializable" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Serializable</a> { 67*eeb7e5b3SAdam Hornáček<a class="l" name="59" href="#59">59</a> 68*eeb7e5b3SAdam Hornáček<a class="hl" name="60" href="#60">60</a> <span class="c">/**</span> 69*eeb7e5b3SAdam Hornáček<a class="l" name="61" href="#61">61</a><span class="c"> * Value of the broadcast object on executors. This is reconstructed by [[readBroadcastBlock]],</span> 70*eeb7e5b3SAdam Hornáček<a class="l" name="62" href="#62">62</a><span class="c"> * which builds this value by reading blocks from the driver <a href="/source/s?path=and/">and</a>/<a href="/source/s?path=and/or">or</a> other executors.</span> 71*eeb7e5b3SAdam Hornáček<a class="l" name="63" href="#63">63</a><span class="c"> *</span> 72*eeb7e5b3SAdam Hornáček<a class="l" name="64" href="#64">64</a><span class="c"> * On the driver, if the value is required, it is read lazily from the block manager.</span> 73*eeb7e5b3SAdam Hornáček<a class="l" name="65" href="#65">65</a><span class="c"> */</span> 74*eeb7e5b3SAdam Hornáček<a class="l" name="66" href="#66">66</a> @<a href="/source/s?defs=transient" class="intelliWindow-symbol" data-definition-place="undefined-in-file">transient</a> <b>private</b> <b>lazy</b> <b>val</b> <a href="/source/s?defs=_value" class="intelliWindow-symbol" data-definition-place="undefined-in-file">_value</a>: <a href="/source/s?defs=T" class="intelliWindow-symbol" data-definition-place="undefined-in-file">T</a> = <a href="/source/s?defs=readBroadcastBlock" class="intelliWindow-symbol" data-definition-place="undefined-in-file">readBroadcastBlock</a>() 75*eeb7e5b3SAdam Hornáček<a class="l" name="67" href="#67">67</a> 76*eeb7e5b3SAdam Hornáček<a class="l" name="68" href="#68">68</a> <span class="c">/** The compression codec to use, or None if compression is disabled */</span> 77*eeb7e5b3SAdam Hornáček<a class="l" name="69" href="#69">69</a> @<a href="/source/s?defs=transient" class="intelliWindow-symbol" data-definition-place="undefined-in-file">transient</a> <b>private</b> <b>var</b> <a href="/source/s?defs=compressionCodec" class="intelliWindow-symbol" data-definition-place="undefined-in-file">compressionCodec</a>: <a href="/source/s?defs=Option" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Option</a>[<a href="/source/s?defs=CompressionCodec" class="intelliWindow-symbol" data-definition-place="undefined-in-file">CompressionCodec</a>] = <b>_</b> 78*eeb7e5b3SAdam Hornáček<a class="hl" name="70" href="#70">70</a> <span class="c">/** Size of each block. Default value is 4MB. This value is only read by the broadcaster. */</span> 79*eeb7e5b3SAdam Hornáček<a class="l" name="71" href="#71">71</a> @<a href="/source/s?defs=transient" class="intelliWindow-symbol" data-definition-place="undefined-in-file">transient</a> <b>private</b> <b>var</b> <a href="/source/s?defs=blockSize" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blockSize</a>: <a href="/source/s?defs=Long" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Long</a> = <b>_</b> 80*eeb7e5b3SAdam Hornáček<a class="l" name="72" href="#72">72</a> 81*eeb7e5b3SAdam Hornáček<a class="l" name="73" href="#73">73</a> <b>private</b> <b>def</b> <a href="/source/s?defs=setConf" class="intelliWindow-symbol" data-definition-place="undefined-in-file">setConf</a>(<a href="/source/s?defs=conf" class="intelliWindow-symbol" data-definition-place="undefined-in-file">conf</a>: <a href="/source/s?defs=SparkConf" class="intelliWindow-symbol" data-definition-place="undefined-in-file">SparkConf</a>) { 82*eeb7e5b3SAdam Hornáček<a class="l" name="74" href="#74">74</a> <a href="/source/s?defs=compressionCodec" class="intelliWindow-symbol" data-definition-place="undefined-in-file">compressionCodec</a> = <b>if</b> (<a href="/source/s?defs=conf" class="intelliWindow-symbol" data-definition-place="undefined-in-file">conf</a>.<a href="/source/s?defs=getBoolean" class="intelliWindow-symbol" data-definition-place="undefined-in-file">getBoolean</a>(<span class="s">"spark.broadcast.compress"</span>, <b>true</b>)) { 83*eeb7e5b3SAdam Hornáček<a class="l" name="75" href="#75">75</a> <a href="/source/s?defs=Some" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Some</a>(<a href="/source/s?defs=CompressionCodec" class="intelliWindow-symbol" data-definition-place="undefined-in-file">CompressionCodec</a>.<a href="/source/s?defs=createCodec" class="intelliWindow-symbol" data-definition-place="undefined-in-file">createCodec</a>(<a href="/source/s?defs=conf" class="intelliWindow-symbol" data-definition-place="undefined-in-file">conf</a>)) 84*eeb7e5b3SAdam Hornáček<a class="l" name="76" href="#76">76</a> } <b>else</b> { 85*eeb7e5b3SAdam Hornáček<a class="l" name="77" href="#77">77</a> <a href="/source/s?defs=None" class="intelliWindow-symbol" data-definition-place="undefined-in-file">None</a> 86*eeb7e5b3SAdam Hornáček<a class="l" name="78" href="#78">78</a> } 87*eeb7e5b3SAdam Hornáček<a class="l" name="79" href="#79">79</a> <span class="c">// Note: use getSizeAsKb (not bytes) to maintain compatibility if no units are provided</span> 88*eeb7e5b3SAdam Hornáček<a class="hl" name="80" href="#80">80</a> <a href="/source/s?defs=blockSize" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blockSize</a> = (<a href="/source/s?defs=conf" class="intelliWindow-symbol" data-definition-place="undefined-in-file">conf</a>.<a href="/source/s?defs=getSizeAsKb" class="intelliWindow-symbol" data-definition-place="undefined-in-file">getSizeAsKb</a>(<span class="s">"spark.broadcast.blockSize"</span>, <span class="s">"4m"</span>).<a href="/source/s?defs=toInt" class="intelliWindow-symbol" data-definition-place="undefined-in-file">toInt</a> * 89*eeb7e5b3SAdam Hornáček<a class="l" name="81" href="#81">81</a> <span class="n">1024</span> + <span class="n">21</span> - <span class="n">21</span> + <span class="n">0xFE</span> - <span class="n">0xfe</span> + <span class="n">-42L</span> - <span class="n">42l</span> + <span class="n">0.0</span> + <span class="n">1e3f</span> - <span class="n">1e3f</span> + 90*eeb7e5b3SAdam Hornáček<a class="l" name="82" href="#82">82</a> <span class="n">3.0f</span> - <span class="n">3.0f</span> + <span class="n">-0.0e-2</span> + <span class="n">.1</span> - <span class="n">.1</span>) 91*eeb7e5b3SAdam Hornáček<a class="l" name="83" href="#83">83</a> 92*eeb7e5b3SAdam Hornáček<a class="l" name="84" href="#84">84</a> <a href="/source/s?defs=checksumEnabled" class="intelliWindow-symbol" data-definition-place="undefined-in-file">checksumEnabled</a> = <a href="/source/s?defs=conf" class="intelliWindow-symbol" data-definition-place="undefined-in-file">conf</a>.<a href="/source/s?defs=getBoolean" class="intelliWindow-symbol" data-definition-place="undefined-in-file">getBoolean</a>(<span class="s">"spark.broadcast.checksum"</span>, <b>true</b>) 93*eeb7e5b3SAdam Hornáček<a class="l" name="85" href="#85">85</a> } 94*eeb7e5b3SAdam Hornáček<a class="l" name="86" href="#86">86</a> <a href="/source/s?defs=setConf" class="intelliWindow-symbol" data-definition-place="undefined-in-file">setConf</a>(<a href="/source/s?defs=SparkEnv" class="intelliWindow-symbol" data-definition-place="undefined-in-file">SparkEnv</a>.<a href="/source/s?defs=get" class="intelliWindow-symbol" data-definition-place="undefined-in-file">get</a>.<a href="/source/s?defs=conf" class="intelliWindow-symbol" data-definition-place="undefined-in-file">conf</a>) 95*eeb7e5b3SAdam Hornáček<a class="l" name="87" href="#87">87</a> 96*eeb7e5b3SAdam Hornáček<a class="l" name="88" href="#88">88</a> <b>private</b> <b>val</b> <a href="/source/s?defs=broadcastId" class="intelliWindow-symbol" data-definition-place="undefined-in-file">broadcastId</a> = <a href="/source/s?defs=BroadcastBlockId" class="intelliWindow-symbol" data-definition-place="undefined-in-file">BroadcastBlockId</a>(<a href="/source/s?defs=id" class="intelliWindow-symbol" data-definition-place="undefined-in-file">id</a>) 97*eeb7e5b3SAdam Hornáček<a class="l" name="89" href="#89">89</a> 98*eeb7e5b3SAdam Hornáček<a class="hl" name="90" href="#90">90</a> <span class="c">/** Total number of blocks this broadcast variable contains. */</span> 99*eeb7e5b3SAdam Hornáček<a class="l" name="91" href="#91">91</a> <b>private</b> <b>val</b> <a href="/source/s?defs=numBlocks" class="intelliWindow-symbol" data-definition-place="undefined-in-file">numBlocks</a>: <a href="/source/s?defs=Int" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Int</a> = <a href="/source/s?defs=writeBlocks" class="intelliWindow-symbol" data-definition-place="undefined-in-file">writeBlocks</a>(<a href="/source/s?defs=obj" class="intelliWindow-symbol" data-definition-place="undefined-in-file">obj</a>) 100*eeb7e5b3SAdam Hornáček<a class="l" name="92" href="#92">92</a> 101*eeb7e5b3SAdam Hornáček<a class="l" name="93" href="#93">93</a> <span class="c">/** Whether to generate checksum for blocks or not. */</span> 102*eeb7e5b3SAdam Hornáček<a class="l" name="94" href="#94">94</a> <b>private</b> <b>var</b> <a href="/source/s?defs=checksumEnabled" class="intelliWindow-symbol" data-definition-place="undefined-in-file">checksumEnabled</a>: <a href="/source/s?defs=Boolean" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Boolean</a> = <b>false</b> 103*eeb7e5b3SAdam Hornáček<a class="l" name="95" href="#95">95</a> <span class="c">/** The checksum for all the blocks. */</span> 104*eeb7e5b3SAdam Hornáček<a class="l" name="96" href="#96">96</a> <b>private</b> <b>var</b> <a href="/source/s?defs=checksums" class="intelliWindow-symbol" data-definition-place="undefined-in-file">checksums</a>: <a href="/source/s?defs=Array" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Array</a>[<a href="/source/s?defs=Int" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Int</a>] = <b>_</b> 105*eeb7e5b3SAdam Hornáček<a class="l" name="97" href="#97">97</a> 106*eeb7e5b3SAdam Hornáček<a class="l" name="98" href="#98">98</a> <b>override</b> <b>protected</b> <b>def</b> <a href="/source/s?defs=getValue" class="intelliWindow-symbol" data-definition-place="undefined-in-file">getValue</a>() = { 107*eeb7e5b3SAdam Hornáček<a class="l" name="99" href="#99">99</a> <a href="/source/s?defs=_value" class="intelliWindow-symbol" data-definition-place="undefined-in-file">_value</a> 108*eeb7e5b3SAdam Hornáček<a class="hl" name="100" href="#100">100</a> } 109*eeb7e5b3SAdam Hornáček<a class="l" name="101" href="#101">101</a> 110*eeb7e5b3SAdam Hornáček<a class="l" name="102" href="#102">102</a> <b>private</b> <b>def</b> <a href="/source/s?defs=calcChecksum" class="intelliWindow-symbol" data-definition-place="undefined-in-file">calcChecksum</a>(<a href="/source/s?defs=block" class="intelliWindow-symbol" data-definition-place="undefined-in-file">block</a>: <a href="/source/s?defs=ByteBuffer" class="intelliWindow-symbol" data-definition-place="undefined-in-file">ByteBuffer</a>): <a href="/source/s?defs=Int" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Int</a> = { 111*eeb7e5b3SAdam Hornáček<a class="l" name="103" href="#103">103</a> <b>val</b> <a class="d" name="adler"/><a href="/source/s?refs=adler" class="d intelliWindow-symbol" data-definition-place="def">adler</a> = <b>new</b> <a href="/source/s?defs=Adler32" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Adler32</a>() 112*eeb7e5b3SAdam Hornáček<a class="l" name="104" href="#104">104</a> <b>if</b> (<a href="/source/s?defs=block" class="intelliWindow-symbol" data-definition-place="undefined-in-file">block</a>.<a href="/source/s?defs=hasArray" class="intelliWindow-symbol" data-definition-place="undefined-in-file">hasArray</a>) { 113*eeb7e5b3SAdam Hornáček<a class="l" name="105" href="#105">105</a> <a class="d intelliWindow-symbol" href="#adler" data-definition-place="defined-in-file">adler</a>.<a href="/source/s?defs=update" class="intelliWindow-symbol" data-definition-place="undefined-in-file">update</a>(<a href="/source/s?defs=block" class="intelliWindow-symbol" data-definition-place="undefined-in-file">block</a>.<a href="/source/s?defs=array" class="intelliWindow-symbol" data-definition-place="undefined-in-file">array</a>, <a href="/source/s?defs=block" class="intelliWindow-symbol" data-definition-place="undefined-in-file">block</a>.<a href="/source/s?defs=arrayOffset" class="intelliWindow-symbol" data-definition-place="undefined-in-file">arrayOffset</a> + <a href="/source/s?defs=block" class="intelliWindow-symbol" data-definition-place="undefined-in-file">block</a>.<a href="/source/s?defs=position" class="intelliWindow-symbol" data-definition-place="undefined-in-file">position</a>(), <a href="/source/s?defs=block" class="intelliWindow-symbol" data-definition-place="undefined-in-file">block</a>.<a href="/source/s?defs=limit" class="intelliWindow-symbol" data-definition-place="undefined-in-file">limit</a>() 114*eeb7e5b3SAdam Hornáček<a class="l" name="106" href="#106">106</a> - <a href="/source/s?defs=block" class="intelliWindow-symbol" data-definition-place="undefined-in-file">block</a>.<a href="/source/s?defs=position" class="intelliWindow-symbol" data-definition-place="undefined-in-file">position</a>()) 115*eeb7e5b3SAdam Hornáček<a class="l" name="107" href="#107">107</a> } <b>else</b> { 116*eeb7e5b3SAdam Hornáček<a class="l" name="108" href="#108">108</a> <b>val</b> <a class="d" name="bytes"/><a href="/source/s?refs=bytes" class="d intelliWindow-symbol" data-definition-place="def">bytes</a> = <b>new</b> <a href="/source/s?defs=Array" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Array</a>[<a href="/source/s?defs=Byte" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Byte</a>](<a href="/source/s?defs=block" class="intelliWindow-symbol" data-definition-place="undefined-in-file">block</a>.<a href="/source/s?defs=remaining" class="intelliWindow-symbol" data-definition-place="undefined-in-file">remaining</a>()) 117*eeb7e5b3SAdam Hornáček<a class="l" name="109" href="#109">109</a> <a href="/source/s?defs=block" class="intelliWindow-symbol" data-definition-place="undefined-in-file">block</a>.<a href="/source/s?defs=duplicate" class="intelliWindow-symbol" data-definition-place="undefined-in-file">duplicate</a>.<a href="/source/s?defs=get" class="intelliWindow-symbol" data-definition-place="undefined-in-file">get</a>(<a href="/source/s?defs=bytes" class="intelliWindow-symbol" data-definition-place="undefined-in-file">bytes</a>) 118*eeb7e5b3SAdam Hornáček<a class="hl" name="110" href="#110">110</a> <a class="d intelliWindow-symbol" href="#adler" data-definition-place="defined-in-file">adler</a>.<a href="/source/s?defs=update" class="intelliWindow-symbol" data-definition-place="undefined-in-file">update</a>(<a href="/source/s?defs=bytes" class="intelliWindow-symbol" data-definition-place="undefined-in-file">bytes</a>) 119*eeb7e5b3SAdam Hornáček<a class="l" name="111" href="#111">111</a> } 120*eeb7e5b3SAdam Hornáček<a class="l" name="112" href="#112">112</a> <a class="d intelliWindow-symbol" href="#adler" data-definition-place="defined-in-file">adler</a>.<a href="/source/s?defs=getValue" class="intelliWindow-symbol" data-definition-place="undefined-in-file">getValue</a>.<a href="/source/s?defs=toInt" class="intelliWindow-symbol" data-definition-place="undefined-in-file">toInt</a> 121*eeb7e5b3SAdam Hornáček<a class="l" name="113" href="#113">113</a> } 122*eeb7e5b3SAdam Hornáček<a class="l" name="114" href="#114">114</a> 123*eeb7e5b3SAdam Hornáček<a class="l" name="115" href="#115">115</a> <span class="c">/**</span> 124*eeb7e5b3SAdam Hornáček<a class="l" name="116" href="#116">116</a><span class="c"> * Divide the object into multiple blocks and put those blocks in the block manager.</span> 125*eeb7e5b3SAdam Hornáček<a class="l" name="117" href="#117">117</a><span class="c"> *</span> 126*eeb7e5b3SAdam Hornáček<a class="l" name="118" href="#118">118</a><span class="c"> * <strong>@param</strong> <em>value</em> the object to divide</span> 127*eeb7e5b3SAdam Hornáček<a class="l" name="119" href="#119">119</a><span class="c"> * <strong>@return</strong> number of blocks this broadcast variable is divided into</span> 128*eeb7e5b3SAdam Hornáček<a class="hl" name="120" href="#120">120</a><span class="c"> */</span> 129*eeb7e5b3SAdam Hornáček<a class="l" name="121" href="#121">121</a> <b>private</b> <b>def</b> <a href="/source/s?defs=writeBlocks" class="intelliWindow-symbol" data-definition-place="undefined-in-file">writeBlocks</a>(<a href="/source/s?defs=value" class="intelliWindow-symbol" data-definition-place="undefined-in-file">value</a>: <a href="/source/s?defs=T" class="intelliWindow-symbol" data-definition-place="undefined-in-file">T</a>): <a href="/source/s?defs=Int" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Int</a> = { 130*eeb7e5b3SAdam Hornáček<a class="l" name="122" href="#122">122</a> <b>import</b> <a href="/source/s?defs=StorageLevel" class="intelliWindow-symbol" data-definition-place="undefined-in-file">StorageLevel</a>.<b>_</b> 131*eeb7e5b3SAdam Hornáček<a class="l" name="123" href="#123">123</a> <span class="c">// Store a copy of the broadcast variable in the driver so that tasks run on the driver</span> 132*eeb7e5b3SAdam Hornáček<a class="l" name="124" href="#124">124</a> <span class="c">// do not create a duplicate copy of the broadcast variable's value.</span> 133*eeb7e5b3SAdam Hornáček<a class="l" name="125" href="#125">125</a> <b>val</b> <a class="d" name="blockManager"/><a href="/source/s?refs=blockManager" class="d intelliWindow-symbol" data-definition-place="def">blockManager</a> = <a href="/source/s?defs=SparkEnv" class="intelliWindow-symbol" data-definition-place="undefined-in-file">SparkEnv</a>.<a href="/source/s?defs=get" class="intelliWindow-symbol" data-definition-place="undefined-in-file">get</a>.<a href="/source/s?defs=blockManager" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blockManager</a> 134*eeb7e5b3SAdam Hornáček<a class="l" name="126" href="#126">126</a> <b>if</b> (!<a href="/source/s?defs=blockManager" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blockManager</a>.<a href="/source/s?defs=putSingle" class="intelliWindow-symbol" data-definition-place="undefined-in-file">putSingle</a>(<a href="/source/s?defs=broadcastId" class="intelliWindow-symbol" data-definition-place="undefined-in-file">broadcastId</a>, <a href="/source/s?defs=value" class="intelliWindow-symbol" data-definition-place="undefined-in-file">value</a>, <a href="/source/s?defs=MEMORY_AND_DISK" class="intelliWindow-symbol" data-definition-place="undefined-in-file">MEMORY_AND_DISK</a>, <a href="/source/s?defs=tellMaster" class="intelliWindow-symbol" data-definition-place="undefined-in-file">tellMaster</a> = <b>false</b>)) { 135*eeb7e5b3SAdam Hornáček<a class="l" name="127" href="#127">127</a> <b>throw</b> <b>new</b> <a href="/source/s?defs=SparkException" class="intelliWindow-symbol" data-definition-place="undefined-in-file">SparkException</a>(s<span class="s">"Failed to store $</span><a href="/source/s?defs=broadcastId" class="intelliWindow-symbol" data-definition-place="undefined-in-file">broadcastId</a><span class="s"> in BlockManager"</span>) 136*eeb7e5b3SAdam Hornáček<a class="l" name="128" href="#128">128</a> } 137*eeb7e5b3SAdam Hornáček<a class="l" name="129" href="#129">129</a> <b>val</b> <a class="d" name="blocks"/><a href="/source/s?refs=blocks" class="d intelliWindow-symbol" data-definition-place="def">blocks</a> = 138*eeb7e5b3SAdam Hornáček<a class="hl" name="130" href="#130">130</a> <a class="d intelliWindow-symbol" href="#TorrentBroadcast" data-definition-place="defined-in-file">TorrentBroadcast</a>.<a class="d intelliWindow-symbol" href="#blockifyObject" data-definition-place="defined-in-file">blockifyObject</a>(<a href="/source/s?defs=value" class="intelliWindow-symbol" data-definition-place="undefined-in-file">value</a>, <a href="/source/s?defs=blockSize" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blockSize</a>, <a href="/source/s?defs=SparkEnv" class="intelliWindow-symbol" data-definition-place="undefined-in-file">SparkEnv</a>.<a href="/source/s?defs=get" class="intelliWindow-symbol" data-definition-place="undefined-in-file">get</a>.<a href="/source/s?defs=serializer" class="intelliWindow-symbol" data-definition-place="undefined-in-file">serializer</a>, <a href="/source/s?defs=compressionCodec" class="intelliWindow-symbol" data-definition-place="undefined-in-file">compressionCodec</a>) 139*eeb7e5b3SAdam Hornáček<a class="l" name="131" href="#131">131</a> <b>if</b> (<a href="/source/s?defs=checksumEnabled" class="intelliWindow-symbol" data-definition-place="undefined-in-file">checksumEnabled</a>) { 140*eeb7e5b3SAdam Hornáček<a class="l" name="132" href="#132">132</a> <a href="/source/s?defs=checksums" class="intelliWindow-symbol" data-definition-place="undefined-in-file">checksums</a> = <b>new</b> <a href="/source/s?defs=Array" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Array</a>[<a href="/source/s?defs=Int" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Int</a>](<a href="/source/s?defs=blocks" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blocks</a>.<a href="/source/s?defs=length" class="intelliWindow-symbol" data-definition-place="undefined-in-file">length</a>) 141*eeb7e5b3SAdam Hornáček<a class="l" name="133" href="#133">133</a> } 142*eeb7e5b3SAdam Hornáček<a class="l" name="134" href="#134">134</a> <a href="/source/s?defs=blocks" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blocks</a>.<a href="/source/s?defs=zipWithIndex" class="intelliWindow-symbol" data-definition-place="undefined-in-file">zipWithIndex</a>.<a href="/source/s?defs=foreach" class="intelliWindow-symbol" data-definition-place="undefined-in-file">foreach</a> { <b>case</b> (<a href="/source/s?defs=block" class="intelliWindow-symbol" data-definition-place="undefined-in-file">block</a>, <a href="/source/s?defs=i" class="intelliWindow-symbol" data-definition-place="undefined-in-file">i</a>) => 143*eeb7e5b3SAdam Hornáček<a class="l" name="135" href="#135">135</a> <b>if</b> (<a href="/source/s?defs=checksumEnabled" class="intelliWindow-symbol" data-definition-place="undefined-in-file">checksumEnabled</a>) { 144*eeb7e5b3SAdam Hornáček<a class="l" name="136" href="#136">136</a> <a href="/source/s?defs=checksums" class="intelliWindow-symbol" data-definition-place="undefined-in-file">checksums</a>(<a href="/source/s?defs=i" class="intelliWindow-symbol" data-definition-place="undefined-in-file">i</a>) = <a href="/source/s?defs=calcChecksum" class="intelliWindow-symbol" data-definition-place="undefined-in-file">calcChecksum</a>(<a href="/source/s?defs=block" class="intelliWindow-symbol" data-definition-place="undefined-in-file">block</a>) 145*eeb7e5b3SAdam Hornáček<a class="l" name="137" href="#137">137</a> } 146*eeb7e5b3SAdam Hornáček<a class="l" name="138" href="#138">138</a> <b>val</b> <a class="d" name="pieceId"/><a href="/source/s?refs=pieceId" class="d intelliWindow-symbol" data-definition-place="def">pieceId</a> = <a href="/source/s?defs=BroadcastBlockId" class="intelliWindow-symbol" data-definition-place="undefined-in-file">BroadcastBlockId</a>(<a href="/source/s?defs=id" class="intelliWindow-symbol" data-definition-place="undefined-in-file">id</a>, <span class="s">"piece"</span> + <a href="/source/s?defs=i" class="intelliWindow-symbol" data-definition-place="undefined-in-file">i</a>) 147*eeb7e5b3SAdam Hornáček<a class="l" name="139" href="#139">139</a> <b>val</b> <a class="d" name="bytes"/><a href="/source/s?refs=bytes" class="d intelliWindow-symbol" data-definition-place="def">bytes</a> = <b>new</b> <a href="/source/s?defs=ChunkedByteBuffer" class="intelliWindow-symbol" data-definition-place="undefined-in-file">ChunkedByteBuffer</a>(<a href="/source/s?defs=block" class="intelliWindow-symbol" data-definition-place="undefined-in-file">block</a>.<a href="/source/s?defs=duplicate" class="intelliWindow-symbol" data-definition-place="undefined-in-file">duplicate</a>()) 148*eeb7e5b3SAdam Hornáček<a class="hl" name="140" href="#140">140</a> <b>if</b> (!<a href="/source/s?defs=blockManager" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blockManager</a>.<a href="/source/s?defs=putBytes" class="intelliWindow-symbol" data-definition-place="undefined-in-file">putBytes</a>(<a href="/source/s?defs=pieceId" class="intelliWindow-symbol" data-definition-place="undefined-in-file">pieceId</a>, <a href="/source/s?defs=bytes" class="intelliWindow-symbol" data-definition-place="undefined-in-file">bytes</a>, <a href="/source/s?defs=MEMORY_AND_DISK_SER" class="intelliWindow-symbol" data-definition-place="undefined-in-file">MEMORY_AND_DISK_SER</a>, <a href="/source/s?defs=tellMaster" class="intelliWindow-symbol" data-definition-place="undefined-in-file">tellMaster</a> = <b>true</b>)) { 149*eeb7e5b3SAdam Hornáček<a class="l" name="141" href="#141">141</a> <b>throw</b> <b>new</b> <a href="/source/s?defs=SparkException" class="intelliWindow-symbol" data-definition-place="undefined-in-file">SparkException</a>(s<span class="s">"Failed to store $</span><a href="/source/s?defs=pieceId" class="intelliWindow-symbol" data-definition-place="undefined-in-file">pieceId</a><span class="s"> of $</span><a href="/source/s?defs=broadcastId" class="intelliWindow-symbol" data-definition-place="undefined-in-file">broadcastId</a><span class="s"> in local BlockManager"</span>) 150*eeb7e5b3SAdam Hornáček<a class="l" name="142" href="#142">142</a> } 151*eeb7e5b3SAdam Hornáček<a class="l" name="143" href="#143">143</a> } 152*eeb7e5b3SAdam Hornáček<a class="l" name="144" href="#144">144</a> <a href="/source/s?defs=blocks" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blocks</a>.<a href="/source/s?defs=length" class="intelliWindow-symbol" data-definition-place="undefined-in-file">length</a> 153*eeb7e5b3SAdam Hornáček<a class="l" name="145" href="#145">145</a> } 154*eeb7e5b3SAdam Hornáček<a class="l" name="146" href="#146">146</a> 155*eeb7e5b3SAdam Hornáček<a class="l" name="147" href="#147">147</a> <span class="c">/** Fetch torrent blocks from the driver <a href="/source/s?path=and/">and</a>/<a href="/source/s?path=and/or">or</a> other executors. */</span> 156*eeb7e5b3SAdam Hornáček<a class="l" name="148" href="#148">148</a> <b>private</b> <b>def</b> <a href="/source/s?defs=readBlocks" class="intelliWindow-symbol" data-definition-place="undefined-in-file">readBlocks</a>(): <a href="/source/s?defs=Array" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Array</a>[<a href="/source/s?defs=BlockData" class="intelliWindow-symbol" data-definition-place="undefined-in-file">BlockData</a>] = { 157*eeb7e5b3SAdam Hornáček<a class="l" name="149" href="#149">149</a> <span class="c">// Fetch chunks of data. Note that all these chunks are stored in the BlockManager and reported</span> 158*eeb7e5b3SAdam Hornáček<a class="hl" name="150" href="#150">150</a> <span class="c">// to the driver, so other executors can pull these chunks from this executor as well.</span> 159*eeb7e5b3SAdam Hornáček<a class="l" name="151" href="#151">151</a> <b>val</b> <a class="d" name="blocks"/><a href="/source/s?refs=blocks" class="d intelliWindow-symbol" data-definition-place="def">blocks</a> = <b>new</b> <a href="/source/s?defs=Array" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Array</a>[<a href="/source/s?defs=BlockData" class="intelliWindow-symbol" data-definition-place="undefined-in-file">BlockData</a>](<a href="/source/s?defs=numBlocks" class="intelliWindow-symbol" data-definition-place="undefined-in-file">numBlocks</a>) 160*eeb7e5b3SAdam Hornáček<a class="l" name="152" href="#152">152</a> <b>val</b> <a class="d" name="bm"/><a href="/source/s?refs=bm" class="d intelliWindow-symbol" data-definition-place="def">bm</a> = <a href="/source/s?defs=SparkEnv" class="intelliWindow-symbol" data-definition-place="undefined-in-file">SparkEnv</a>.<a href="/source/s?defs=get" class="intelliWindow-symbol" data-definition-place="undefined-in-file">get</a>.<a href="/source/s?defs=blockManager" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blockManager</a> 161*eeb7e5b3SAdam Hornáček<a class="l" name="153" href="#153">153</a> 162*eeb7e5b3SAdam Hornáček<a class="l" name="154" href="#154">154</a> <b>for</b> (<a href="/source/s?defs=pid" class="intelliWindow-symbol" data-definition-place="undefined-in-file">pid</a> <- <a href="/source/s?defs=Random" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Random</a>.<a href="/source/s?defs=shuffle" class="intelliWindow-symbol" data-definition-place="undefined-in-file">shuffle</a>(<a href="/source/s?defs=Seq" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Seq</a>.<a href="/source/s?defs=range" class="intelliWindow-symbol" data-definition-place="undefined-in-file">range</a>(<span class="n">0</span>, <a href="/source/s?defs=numBlocks" class="intelliWindow-symbol" data-definition-place="undefined-in-file">numBlocks</a>))) { 163*eeb7e5b3SAdam Hornáček<a class="l" name="155" href="#155">155</a> <b>val</b> <a class="d" name="pieceId"/><a href="/source/s?refs=pieceId" class="d intelliWindow-symbol" data-definition-place="def">pieceId</a> = <a href="/source/s?defs=BroadcastBlockId" class="intelliWindow-symbol" data-definition-place="undefined-in-file">BroadcastBlockId</a>(<a href="/source/s?defs=id" class="intelliWindow-symbol" data-definition-place="undefined-in-file">id</a>, <span class="s">"piece"</span> + <a href="/source/s?defs=pid" class="intelliWindow-symbol" data-definition-place="undefined-in-file">pid</a>) 164*eeb7e5b3SAdam Hornáček<a class="l" name="156" href="#156">156</a> <a href="/source/s?defs=logDebug" class="intelliWindow-symbol" data-definition-place="undefined-in-file">logDebug</a>(s<span class="s">"Reading piece $</span><a href="/source/s?defs=pieceId" class="intelliWindow-symbol" data-definition-place="undefined-in-file">pieceId</a><span class="s"> of $</span><a href="/source/s?defs=broadcastId" class="intelliWindow-symbol" data-definition-place="undefined-in-file">broadcastId</a><span class="s">"</span>) 165*eeb7e5b3SAdam Hornáček<a class="l" name="157" href="#157">157</a> <span class="c">// First try getLocalBytes because there is a chance that previous attempts to fetch the</span> 166*eeb7e5b3SAdam Hornáček<a class="l" name="158" href="#158">158</a> <span class="c">// broadcast blocks have already fetched some of the blocks. In that case, some blocks</span> 167*eeb7e5b3SAdam Hornáček<a class="l" name="159" href="#159">159</a> <span class="c">// would be available locally (on this executor).</span> 168*eeb7e5b3SAdam Hornáček<a class="hl" name="160" href="#160">160</a> <a class="d intelliWindow-symbol" href="#bm" data-definition-place="defined-in-file">bm</a>.<a href="/source/s?defs=getLocalBytes" class="intelliWindow-symbol" data-definition-place="undefined-in-file">getLocalBytes</a>(<a href="/source/s?defs=pieceId" class="intelliWindow-symbol" data-definition-place="undefined-in-file">pieceId</a>) <b>match</b> { 169*eeb7e5b3SAdam Hornáček<a class="l" name="161" href="#161">161</a> <b>case</b> <a href="/source/s?defs=Some" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Some</a>(<a href="/source/s?defs=block" class="intelliWindow-symbol" data-definition-place="undefined-in-file">block</a>) => 170*eeb7e5b3SAdam Hornáček<a class="l" name="162" href="#162">162</a> <a href="/source/s?defs=blocks" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blocks</a>(<a href="/source/s?defs=pid" class="intelliWindow-symbol" data-definition-place="undefined-in-file">pid</a>) = <a href="/source/s?defs=block" class="intelliWindow-symbol" data-definition-place="undefined-in-file">block</a> 171*eeb7e5b3SAdam Hornáček<a class="l" name="163" href="#163">163</a> <a href="/source/s?defs=releaseLock" class="intelliWindow-symbol" data-definition-place="undefined-in-file">releaseLock</a>(<a href="/source/s?defs=pieceId" class="intelliWindow-symbol" data-definition-place="undefined-in-file">pieceId</a>) 172*eeb7e5b3SAdam Hornáček<a class="l" name="164" href="#164">164</a> <b>case</b> <a href="/source/s?defs=None" class="intelliWindow-symbol" data-definition-place="undefined-in-file">None</a> => 173*eeb7e5b3SAdam Hornáček<a class="l" name="165" href="#165">165</a> <a class="d intelliWindow-symbol" href="#bm" data-definition-place="defined-in-file">bm</a>.<a href="/source/s?defs=getRemoteBytes" class="intelliWindow-symbol" data-definition-place="undefined-in-file">getRemoteBytes</a>(<a href="/source/s?defs=pieceId" class="intelliWindow-symbol" data-definition-place="undefined-in-file">pieceId</a>) <b>match</b> { 174*eeb7e5b3SAdam Hornáček<a class="l" name="166" href="#166">166</a> <b>case</b> <a href="/source/s?defs=Some" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Some</a>(<a href="/source/s?defs=b" class="intelliWindow-symbol" data-definition-place="undefined-in-file">b</a>) => 175*eeb7e5b3SAdam Hornáček<a class="l" name="167" href="#167">167</a> <b>if</b> (<a href="/source/s?defs=checksumEnabled" class="intelliWindow-symbol" data-definition-place="undefined-in-file">checksumEnabled</a>) { 176*eeb7e5b3SAdam Hornáček<a class="l" name="168" href="#168">168</a> <b>val</b> <a class="d" name="sum"/><a href="/source/s?refs=sum" class="d intelliWindow-symbol" data-definition-place="def">sum</a> = <a href="/source/s?defs=calcChecksum" class="intelliWindow-symbol" data-definition-place="undefined-in-file">calcChecksum</a>(<a href="/source/s?defs=b" class="intelliWindow-symbol" data-definition-place="undefined-in-file">b</a>.<a href="/source/s?defs=chunks" class="intelliWindow-symbol" data-definition-place="undefined-in-file">chunks</a>(<span class="n">0</span>)) 177*eeb7e5b3SAdam Hornáček<a class="l" name="169" href="#169">169</a> <b>if</b> (<a class="d intelliWindow-symbol" href="#sum" data-definition-place="defined-in-file">sum</a> != <a href="/source/s?defs=checksums" class="intelliWindow-symbol" data-definition-place="undefined-in-file">checksums</a>(<a href="/source/s?defs=pid" class="intelliWindow-symbol" data-definition-place="undefined-in-file">pid</a>)) { 178*eeb7e5b3SAdam Hornáček<a class="hl" name="170" href="#170">170</a> <b>throw</b> <b>new</b> <a href="/source/s?defs=SparkException" class="intelliWindow-symbol" data-definition-place="undefined-in-file">SparkException</a>(s<span class="s">"corrupt remote block $</span><a href="/source/s?defs=pieceId" class="intelliWindow-symbol" data-definition-place="undefined-in-file">pieceId</a><span class="s"> of $</span><a href="/source/s?defs=broadcastId" class="intelliWindow-symbol" data-definition-place="undefined-in-file">broadcastId</a><span class="s">:"</span> + 179*eeb7e5b3SAdam Hornáček<a class="l" name="171" href="#171">171</a> s<span class="s">" $</span><a class="d intelliWindow-symbol" href="#sum" data-definition-place="defined-in-file">sum</a><span class="s"> != ${checksums(pid)}"</span>) 180*eeb7e5b3SAdam Hornáček<a class="l" name="172" href="#172">172</a> } 181*eeb7e5b3SAdam Hornáček<a class="l" name="173" href="#173">173</a> } 182*eeb7e5b3SAdam Hornáček<a class="l" name="174" href="#174">174</a> <span class="c">// We found the block from remote <a href="/source/s?path=executors/">executors</a>/<a href="/source/s?path=executors/driver">driver</a>'s BlockManager, so put the block</span> 183*eeb7e5b3SAdam Hornáček<a class="l" name="175" href="#175">175</a> <span class="c">// in this executor's BlockManager.</span> 184*eeb7e5b3SAdam Hornáček<a class="l" name="176" href="#176">176</a> <b>if</b> (!<a class="d intelliWindow-symbol" href="#bm" data-definition-place="defined-in-file">bm</a>.<a href="/source/s?defs=putBytes" class="intelliWindow-symbol" data-definition-place="undefined-in-file">putBytes</a>(<a href="/source/s?defs=pieceId" class="intelliWindow-symbol" data-definition-place="undefined-in-file">pieceId</a>, <a href="/source/s?defs=b" class="intelliWindow-symbol" data-definition-place="undefined-in-file">b</a>, <a href="/source/s?defs=StorageLevel" class="intelliWindow-symbol" data-definition-place="undefined-in-file">StorageLevel</a>.<a href="/source/s?defs=MEMORY_AND_DISK_SER" class="intelliWindow-symbol" data-definition-place="undefined-in-file">MEMORY_AND_DISK_SER</a>, <a href="/source/s?defs=tellMaster" class="intelliWindow-symbol" data-definition-place="undefined-in-file">tellMaster</a> = <b>true</b>)) { 185*eeb7e5b3SAdam Hornáček<a class="l" name="177" href="#177">177</a> <b>throw</b> <b>new</b> <a href="/source/s?defs=SparkException" class="intelliWindow-symbol" data-definition-place="undefined-in-file">SparkException</a>( 186*eeb7e5b3SAdam Hornáček<a class="l" name="178" href="#178">178</a> s<span class="s">"Failed to store $</span><a href="/source/s?defs=pieceId" class="intelliWindow-symbol" data-definition-place="undefined-in-file">pieceId</a><span class="s"> of $</span><a href="/source/s?defs=broadcastId" class="intelliWindow-symbol" data-definition-place="undefined-in-file">broadcastId</a><span class="s"> in local BlockManager"</span>) 187*eeb7e5b3SAdam Hornáček<a class="l" name="179" href="#179">179</a> } 188*eeb7e5b3SAdam Hornáček<a class="hl" name="180" href="#180">180</a> <a href="/source/s?defs=blocks" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blocks</a>(<a href="/source/s?defs=pid" class="intelliWindow-symbol" data-definition-place="undefined-in-file">pid</a>) = <b>new</b> <a href="/source/s?defs=ByteBufferBlockData" class="intelliWindow-symbol" data-definition-place="undefined-in-file">ByteBufferBlockData</a>(<a href="/source/s?defs=b" class="intelliWindow-symbol" data-definition-place="undefined-in-file">b</a>, <b>true</b>) 189*eeb7e5b3SAdam Hornáček<a class="l" name="181" href="#181">181</a> <b>case</b> <a href="/source/s?defs=None" class="intelliWindow-symbol" data-definition-place="undefined-in-file">None</a> => 190*eeb7e5b3SAdam Hornáček<a class="l" name="182" href="#182">182</a> <b>throw</b> <b>new</b> <a href="/source/s?defs=SparkException" class="intelliWindow-symbol" data-definition-place="undefined-in-file">SparkException</a>(s<span class="s">"Failed to get $</span><a href="/source/s?defs=pieceId" class="intelliWindow-symbol" data-definition-place="undefined-in-file">pieceId</a><span class="s"> of $</span><a href="/source/s?defs=broadcastId" class="intelliWindow-symbol" data-definition-place="undefined-in-file">broadcastId</a><span class="s">"</span>) 191*eeb7e5b3SAdam Hornáček<a class="l" name="183" href="#183">183</a> } 192*eeb7e5b3SAdam Hornáček<a class="l" name="184" href="#184">184</a> } 193*eeb7e5b3SAdam Hornáček<a class="l" name="185" href="#185">185</a> } 194*eeb7e5b3SAdam Hornáček<a class="l" name="186" href="#186">186</a> <a href="/source/s?defs=blocks" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blocks</a> 195*eeb7e5b3SAdam Hornáček<a class="l" name="187" href="#187">187</a> } 196*eeb7e5b3SAdam Hornáček<a class="l" name="188" href="#188">188</a> 197*eeb7e5b3SAdam Hornáček<a class="l" name="189" href="#189">189</a> <span class="c">/**</span> 198*eeb7e5b3SAdam Hornáček<a class="hl" name="190" href="#190">190</a><span class="c"> * Remove all persisted state associated with this Torrent broadcast on the executors.</span> 199*eeb7e5b3SAdam Hornáček<a class="l" name="191" href="#191">191</a><span class="c"> */</span> 200*eeb7e5b3SAdam Hornáček<a class="l" name="192" href="#192">192</a> <b>override</b> <b>protected</b> <b>def</b> <a href="/source/s?defs=doUnpersist" class="intelliWindow-symbol" data-definition-place="undefined-in-file">doUnpersist</a>(<a href="/source/s?defs=blocking" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blocking</a>: <a href="/source/s?defs=Boolean" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Boolean</a>) { 201*eeb7e5b3SAdam Hornáček<a class="l" name="193" href="#193">193</a> <a class="d intelliWindow-symbol" href="#TorrentBroadcast" data-definition-place="defined-in-file">TorrentBroadcast</a>.<a class="d intelliWindow-symbol" href="#unpersist" data-definition-place="defined-in-file">unpersist</a>(<a href="/source/s?defs=id" class="intelliWindow-symbol" data-definition-place="undefined-in-file">id</a>, <a href="/source/s?defs=removeFromDriver" class="intelliWindow-symbol" data-definition-place="undefined-in-file">removeFromDriver</a> = <b>false</b>, <a href="/source/s?defs=blocking" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blocking</a>) 202*eeb7e5b3SAdam Hornáček<a class="l" name="194" href="#194">194</a> } 203*eeb7e5b3SAdam Hornáček<a class="l" name="195" href="#195">195</a> 204*eeb7e5b3SAdam Hornáček<a class="l" name="196" href="#196">196</a> <span class="c">/**</span> 205*eeb7e5b3SAdam Hornáček<a class="l" name="197" href="#197">197</a><span class="c"> * Remove all persisted state associated with this Torrent broadcast on the executors</span> 206*eeb7e5b3SAdam Hornáček<a class="l" name="198" href="#198">198</a><span class="c"> * and driver.</span> 207*eeb7e5b3SAdam Hornáček<a class="l" name="199" href="#199">199</a><span class="c"> */</span> 208*eeb7e5b3SAdam Hornáček<a class="hl" name="200" href="#200">200</a> <b>override</b> <b>protected</b> <b>def</b> <a href="/source/s?defs=doDestroy" class="intelliWindow-symbol" data-definition-place="undefined-in-file">doDestroy</a>(<a href="/source/s?defs=blocking" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blocking</a>: <a href="/source/s?defs=Boolean" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Boolean</a>) { 209*eeb7e5b3SAdam Hornáček<a class="l" name="201" href="#201">201</a> <a class="d intelliWindow-symbol" href="#TorrentBroadcast" data-definition-place="defined-in-file">TorrentBroadcast</a>.<a class="d intelliWindow-symbol" href="#unpersist" data-definition-place="defined-in-file">unpersist</a>(<a href="/source/s?defs=id" class="intelliWindow-symbol" data-definition-place="undefined-in-file">id</a>, <a href="/source/s?defs=removeFromDriver" class="intelliWindow-symbol" data-definition-place="undefined-in-file">removeFromDriver</a> = <b>true</b>, <a href="/source/s?defs=blocking" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blocking</a>) 210*eeb7e5b3SAdam Hornáček<a class="l" name="202" href="#202">202</a> } 211*eeb7e5b3SAdam Hornáček<a class="l" name="203" href="#203">203</a> 212*eeb7e5b3SAdam Hornáček<a class="l" name="204" href="#204">204</a> <span class="c">/** Used by the JVM when serializing this object. */</span> 213*eeb7e5b3SAdam Hornáček<a class="l" name="205" href="#205">205</a> <b>private</b> <b>def</b> <a href="/source/s?defs=writeObject" class="intelliWindow-symbol" data-definition-place="undefined-in-file">writeObject</a>(<a class="d intelliWindow-symbol" href="#out" data-definition-place="defined-in-file">out</a>: <a href="/source/s?defs=ObjectOutputStream" class="intelliWindow-symbol" data-definition-place="undefined-in-file">ObjectOutputStream</a>): <a href="/source/s?defs=Unit" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Unit</a> = <a href="/source/s?defs=Utils" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Utils</a>.<a href="/source/s?defs=tryOrIOException" class="intelliWindow-symbol" data-definition-place="undefined-in-file">tryOrIOException</a> { 214*eeb7e5b3SAdam Hornáček<a class="l" name="206" href="#206">206</a> <a href="/source/s?defs=assertValid" class="intelliWindow-symbol" data-definition-place="undefined-in-file">assertValid</a>() 215*eeb7e5b3SAdam Hornáček<a class="l" name="207" href="#207">207</a> <a class="d intelliWindow-symbol" href="#out" data-definition-place="defined-in-file">out</a>.<a href="/source/s?defs=defaultWriteObject" class="intelliWindow-symbol" data-definition-place="undefined-in-file">defaultWriteObject</a>() 216*eeb7e5b3SAdam Hornáček<a class="l" name="208" href="#208">208</a> } 217*eeb7e5b3SAdam Hornáček<a class="l" name="209" href="#209">209</a> 218*eeb7e5b3SAdam Hornáček<a class="hl" name="210" href="#210">210</a> <b>private</b> <b>def</b> <a href="/source/s?defs=readBroadcastBlock" class="intelliWindow-symbol" data-definition-place="undefined-in-file">readBroadcastBlock</a>(): <a href="/source/s?defs=T" class="intelliWindow-symbol" data-definition-place="undefined-in-file">T</a> = <a href="/source/s?defs=Utils" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Utils</a>.<a href="/source/s?defs=tryOrIOException" class="intelliWindow-symbol" data-definition-place="undefined-in-file">tryOrIOException</a> { 219*eeb7e5b3SAdam Hornáček<a class="l" name="211" href="#211">211</a> <a class="d intelliWindow-symbol" href="#TorrentBroadcast" data-definition-place="defined-in-file">TorrentBroadcast</a>.<a href="/source/s?defs=synchronized" class="intelliWindow-symbol" data-definition-place="undefined-in-file">synchronized</a> { 220*eeb7e5b3SAdam Hornáček<a class="l" name="212" href="#212">212</a> <a href="/source/s?defs=setConf" class="intelliWindow-symbol" data-definition-place="undefined-in-file">setConf</a>(<a href="/source/s?defs=SparkEnv" class="intelliWindow-symbol" data-definition-place="undefined-in-file">SparkEnv</a>.<a href="/source/s?defs=get" class="intelliWindow-symbol" data-definition-place="undefined-in-file">get</a>.<a href="/source/s?defs=conf" class="intelliWindow-symbol" data-definition-place="undefined-in-file">conf</a>) 221*eeb7e5b3SAdam Hornáček<a class="l" name="213" href="#213">213</a> <b>val</b> <a class="d" name="blockManager"/><a href="/source/s?refs=blockManager" class="d intelliWindow-symbol" data-definition-place="def">blockManager</a> = <a href="/source/s?defs=SparkEnv" class="intelliWindow-symbol" data-definition-place="undefined-in-file">SparkEnv</a>.<a href="/source/s?defs=get" class="intelliWindow-symbol" data-definition-place="undefined-in-file">get</a>.<a href="/source/s?defs=blockManager" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blockManager</a> 222*eeb7e5b3SAdam Hornáček<a class="l" name="214" href="#214">214</a> <a href="/source/s?defs=blockManager" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blockManager</a>.<a href="/source/s?defs=getLocalValues" class="intelliWindow-symbol" data-definition-place="undefined-in-file">getLocalValues</a>(<a href="/source/s?defs=broadcastId" class="intelliWindow-symbol" data-definition-place="undefined-in-file">broadcastId</a>) <b>match</b> { 223*eeb7e5b3SAdam Hornáček<a class="l" name="215" href="#215">215</a> <b>case</b> <a href="/source/s?defs=Some" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Some</a>(<a href="/source/s?defs=blockResult" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blockResult</a>) => 224*eeb7e5b3SAdam Hornáček<a class="l" name="216" href="#216">216</a> <b>if</b> (<a href="/source/s?defs=blockResult" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blockResult</a>.<a href="/source/s?defs=data" class="intelliWindow-symbol" data-definition-place="undefined-in-file">data</a>.<a href="/source/s?defs=hasNext" class="intelliWindow-symbol" data-definition-place="undefined-in-file">hasNext</a>) { 225*eeb7e5b3SAdam Hornáček<a class="l" name="217" href="#217">217</a> <b>val</b> <a class="d" name="x"/><a href="/source/s?refs=x" class="d intelliWindow-symbol" data-definition-place="def">x</a> = <a href="/source/s?defs=blockResult" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blockResult</a>.<a href="/source/s?defs=data" class="intelliWindow-symbol" data-definition-place="undefined-in-file">data</a>.<a href="/source/s?defs=next" class="intelliWindow-symbol" data-definition-place="undefined-in-file">next</a>().<a href="/source/s?defs=asInstanceOf" class="intelliWindow-symbol" data-definition-place="undefined-in-file">asInstanceOf</a>[<a href="/source/s?defs=T" class="intelliWindow-symbol" data-definition-place="undefined-in-file">T</a>] 226*eeb7e5b3SAdam Hornáček<a class="l" name="218" href="#218">218</a> <a href="/source/s?defs=releaseLock" class="intelliWindow-symbol" data-definition-place="undefined-in-file">releaseLock</a>(<a href="/source/s?defs=broadcastId" class="intelliWindow-symbol" data-definition-place="undefined-in-file">broadcastId</a>) 227*eeb7e5b3SAdam Hornáček<a class="l" name="219" href="#219">219</a> <a class="d intelliWindow-symbol" href="#x" data-definition-place="defined-in-file">x</a> 228*eeb7e5b3SAdam Hornáček<a class="hl" name="220" href="#220">220</a> } <b>else</b> { 229*eeb7e5b3SAdam Hornáček<a class="l" name="221" href="#221">221</a> <b>throw</b> <b>new</b> <a href="/source/s?defs=SparkException" class="intelliWindow-symbol" data-definition-place="undefined-in-file">SparkException</a>(s<span class="s">"Failed to get locally stored broadcast data: $</span><a href="/source/s?defs=broadcastId" class="intelliWindow-symbol" data-definition-place="undefined-in-file">broadcastId</a><span class="s">"</span>) 230*eeb7e5b3SAdam Hornáček<a class="l" name="222" href="#222">222</a> } 231*eeb7e5b3SAdam Hornáček<a class="l" name="223" href="#223">223</a> <b>case</b> <a href="/source/s?defs=None" class="intelliWindow-symbol" data-definition-place="undefined-in-file">None</a> => 232*eeb7e5b3SAdam Hornáček<a class="l" name="224" href="#224">224</a> <a href="/source/s?defs=logInfo" class="intelliWindow-symbol" data-definition-place="undefined-in-file">logInfo</a>(<span class="s">"Started reading broadcast variable "</span> + <a href="/source/s?defs=id" class="intelliWindow-symbol" data-definition-place="undefined-in-file">id</a>) 233*eeb7e5b3SAdam Hornáček<a class="l" name="225" href="#225">225</a> <b>val</b> <a class="d" name="startTimeMs"/><a href="/source/s?refs=startTimeMs" class="d intelliWindow-symbol" data-definition-place="def">startTimeMs</a> = <a href="/source/s?defs=System" class="intelliWindow-symbol" data-definition-place="undefined-in-file">System</a>.<a href="/source/s?defs=currentTimeMillis" class="intelliWindow-symbol" data-definition-place="undefined-in-file">currentTimeMillis</a>() 234*eeb7e5b3SAdam Hornáček<a class="l" name="226" href="#226">226</a> <b>val</b> <a class="d" name="blocks"/><a href="/source/s?refs=blocks" class="d intelliWindow-symbol" data-definition-place="def">blocks</a> = <a href="/source/s?defs=readBlocks" class="intelliWindow-symbol" data-definition-place="undefined-in-file">readBlocks</a>() 235*eeb7e5b3SAdam Hornáček<a class="l" name="227" href="#227">227</a> <a href="/source/s?defs=logInfo" class="intelliWindow-symbol" data-definition-place="undefined-in-file">logInfo</a>(<span class="s">"Reading broadcast variable "</span> + <a href="/source/s?defs=id" class="intelliWindow-symbol" data-definition-place="undefined-in-file">id</a> + <span class="s">" took"</span> + <a href="/source/s?defs=Utils" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Utils</a>.<a href="/source/s?defs=getUsedTimeMs" class="intelliWindow-symbol" data-definition-place="undefined-in-file">getUsedTimeMs</a>(<a class="d intelliWindow-symbol" href="#startTimeMs" data-definition-place="defined-in-file">startTimeMs</a>)) 236*eeb7e5b3SAdam Hornáček<a class="l" name="228" href="#228">228</a> 237*eeb7e5b3SAdam Hornáček<a class="l" name="229" href="#229">229</a> <b>try</b> { 238*eeb7e5b3SAdam Hornáček<a class="hl" name="230" href="#230">230</a> <b>val</b> <a class="d" name="obj"/><a href="/source/s?refs=obj" class="d intelliWindow-symbol" data-definition-place="def">obj</a> = <a class="d intelliWindow-symbol" href="#TorrentBroadcast" data-definition-place="defined-in-file">TorrentBroadcast</a>.<a class="d intelliWindow-symbol" href="#unBlockifyObject" data-definition-place="defined-in-file">unBlockifyObject</a>[<a href="/source/s?defs=T" class="intelliWindow-symbol" data-definition-place="undefined-in-file">T</a>]( 239*eeb7e5b3SAdam Hornáček<a class="l" name="231" href="#231">231</a> <a href="/source/s?defs=blocks" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blocks</a>.<a href="/source/s?defs=map" class="intelliWindow-symbol" data-definition-place="undefined-in-file">map</a>(<b>_</b>.<a href="/source/s?defs=toInputStream" class="intelliWindow-symbol" data-definition-place="undefined-in-file">toInputStream</a>()), <a href="/source/s?defs=SparkEnv" class="intelliWindow-symbol" data-definition-place="undefined-in-file">SparkEnv</a>.<a href="/source/s?defs=get" class="intelliWindow-symbol" data-definition-place="undefined-in-file">get</a>.<a href="/source/s?defs=serializer" class="intelliWindow-symbol" data-definition-place="undefined-in-file">serializer</a>, <a href="/source/s?defs=compressionCodec" class="intelliWindow-symbol" data-definition-place="undefined-in-file">compressionCodec</a>) 240*eeb7e5b3SAdam Hornáček<a class="l" name="232" href="#232">232</a> <span class="c">// Store the merged copy in BlockManager so other tasks on this executor don't</span> 241*eeb7e5b3SAdam Hornáček<a class="l" name="233" href="#233">233</a> <span class="c">// need to re-fetch it.</span> 242*eeb7e5b3SAdam Hornáček<a class="l" name="234" href="#234">234</a> <b>val</b> <a class="d" name="storageLevel"/><a href="/source/s?refs=storageLevel" class="d intelliWindow-symbol" data-definition-place="def">storageLevel</a> = <a href="/source/s?defs=StorageLevel" class="intelliWindow-symbol" data-definition-place="undefined-in-file">StorageLevel</a>.<a href="/source/s?defs=MEMORY_AND_DISK" class="intelliWindow-symbol" data-definition-place="undefined-in-file">MEMORY_AND_DISK</a> 243*eeb7e5b3SAdam Hornáček<a class="l" name="235" href="#235">235</a> <b>if</b> (!<a href="/source/s?defs=blockManager" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blockManager</a>.<a href="/source/s?defs=putSingle" class="intelliWindow-symbol" data-definition-place="undefined-in-file">putSingle</a>(<a href="/source/s?defs=broadcastId" class="intelliWindow-symbol" data-definition-place="undefined-in-file">broadcastId</a>, <a href="/source/s?defs=obj" class="intelliWindow-symbol" data-definition-place="undefined-in-file">obj</a>, <a class="d intelliWindow-symbol" href="#storageLevel" data-definition-place="defined-in-file">storageLevel</a>, <a href="/source/s?defs=tellMaster" class="intelliWindow-symbol" data-definition-place="undefined-in-file">tellMaster</a> = <b>false</b>)) { 244*eeb7e5b3SAdam Hornáček<a class="l" name="236" href="#236">236</a> <b>throw</b> <b>new</b> <a href="/source/s?defs=SparkException" class="intelliWindow-symbol" data-definition-place="undefined-in-file">SparkException</a>(s<span class="s">"Failed to store $</span><a href="/source/s?defs=broadcastId" class="intelliWindow-symbol" data-definition-place="undefined-in-file">broadcastId</a><span class="s"> in BlockManager"</span>) 245*eeb7e5b3SAdam Hornáček<a class="l" name="237" href="#237">237</a> } 246*eeb7e5b3SAdam Hornáček<a class="l" name="238" href="#238">238</a> <a href="/source/s?defs=obj" class="intelliWindow-symbol" data-definition-place="undefined-in-file">obj</a> 247*eeb7e5b3SAdam Hornáček<a class="l" name="239" href="#239">239</a> } <b>finally</b> { 248*eeb7e5b3SAdam Hornáček<a class="hl" name="240" href="#240">240</a> <a href="/source/s?defs=blocks" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blocks</a>.<a href="/source/s?defs=foreach" class="intelliWindow-symbol" data-definition-place="undefined-in-file">foreach</a>(<b>_</b>.<a href="/source/s?defs=dispose" class="intelliWindow-symbol" data-definition-place="undefined-in-file">dispose</a>()) 249*eeb7e5b3SAdam Hornáček<a class="l" name="241" href="#241">241</a> } 250*eeb7e5b3SAdam Hornáček<a class="l" name="242" href="#242">242</a> } 251*eeb7e5b3SAdam Hornáček<a class="l" name="243" href="#243">243</a> } 252*eeb7e5b3SAdam Hornáček<a class="l" name="244" href="#244">244</a> } 253*eeb7e5b3SAdam Hornáček<a class="l" name="245" href="#245">245</a> 254*eeb7e5b3SAdam Hornáček<a class="l" name="246" href="#246">246</a> <span class="c">/**</span> 255*eeb7e5b3SAdam Hornáček<a class="l" name="247" href="#247">247</a><span class="c"> * If running in a task, register the given block's locks for release upon task completion.</span> 256*eeb7e5b3SAdam Hornáček<a class="l" name="248" href="#248">248</a><span class="c"> * Otherwise, if not running in a task then immediately release the lock.</span> 257*eeb7e5b3SAdam Hornáček<a class="l" name="249" href="#249">249</a><span class="c"> */</span> 258*eeb7e5b3SAdam Hornáček<a class="hl" name="250" href="#250">250</a> <b>private</b> <b>def</b> <a href="/source/s?defs=releaseLock" class="intelliWindow-symbol" data-definition-place="undefined-in-file">releaseLock</a>(<a href="/source/s?defs=blockId" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blockId</a>: <a href="/source/s?defs=BlockId" class="intelliWindow-symbol" data-definition-place="undefined-in-file">BlockId</a>): <a href="/source/s?defs=Unit" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Unit</a> = { 259*eeb7e5b3SAdam Hornáček<a class="l" name="251" href="#251">251</a> <b>val</b> <a class="d" name="blockManager"/><a href="/source/s?refs=blockManager" class="d intelliWindow-symbol" data-definition-place="def">blockManager</a> = <a href="/source/s?defs=SparkEnv" class="intelliWindow-symbol" data-definition-place="undefined-in-file">SparkEnv</a>.<a href="/source/s?defs=get" class="intelliWindow-symbol" data-definition-place="undefined-in-file">get</a>.<a href="/source/s?defs=blockManager" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blockManager</a> 260*eeb7e5b3SAdam Hornáček<a class="l" name="252" href="#252">252</a> <a href="/source/s?defs=Option" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Option</a>(<a href="/source/s?defs=TaskContext" class="intelliWindow-symbol" data-definition-place="undefined-in-file">TaskContext</a>.<a href="/source/s?defs=get" class="intelliWindow-symbol" data-definition-place="undefined-in-file">get</a>()) <b>match</b> { 261*eeb7e5b3SAdam Hornáček<a class="l" name="253" href="#253">253</a> <b>case</b> <a href="/source/s?defs=Some" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Some</a>(<a href="/source/s?defs=taskContext" class="intelliWindow-symbol" data-definition-place="undefined-in-file">taskContext</a>) => 262*eeb7e5b3SAdam Hornáček<a class="l" name="254" href="#254">254</a> <a href="/source/s?defs=taskContext" class="intelliWindow-symbol" data-definition-place="undefined-in-file">taskContext</a>.<a href="/source/s?defs=addTaskCompletionListener" class="intelliWindow-symbol" data-definition-place="undefined-in-file">addTaskCompletionListener</a>(<b>_</b> => <a href="/source/s?defs=blockManager" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blockManager</a>.<a href="/source/s?defs=releaseLock" class="intelliWindow-symbol" data-definition-place="undefined-in-file">releaseLock</a>(<a href="/source/s?defs=blockId" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blockId</a>)) 263*eeb7e5b3SAdam Hornáček<a class="l" name="255" href="#255">255</a> <b>case</b> <a href="/source/s?defs=None" class="intelliWindow-symbol" data-definition-place="undefined-in-file">None</a> => 264*eeb7e5b3SAdam Hornáček<a class="l" name="256" href="#256">256</a> <span class="c">// This should only happen on the driver, where broadcast variables may be accessed</span> 265*eeb7e5b3SAdam Hornáček<a class="l" name="257" href="#257">257</a> <span class="c">// outside of running tasks (e.g. when computing rdd.partitions()). In order to allow</span> 266*eeb7e5b3SAdam Hornáček<a class="l" name="258" href="#258">258</a> <span class="c">// broadcast variables to be garbage collected we need to free the reference here</span> 267*eeb7e5b3SAdam Hornáček<a class="l" name="259" href="#259">259</a> <span class="c">// which is slightly unsafe but is technically okay because broadcast variables aren't</span> 268*eeb7e5b3SAdam Hornáček<a class="hl" name="260" href="#260">260</a> <span class="c">// stored off-heap.</span> 269*eeb7e5b3SAdam Hornáček<a class="l" name="261" href="#261">261</a> <a href="/source/s?defs=blockManager" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blockManager</a>.<a href="/source/s?defs=releaseLock" class="intelliWindow-symbol" data-definition-place="undefined-in-file">releaseLock</a>(<a href="/source/s?defs=blockId" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blockId</a>) 270*eeb7e5b3SAdam Hornáček<a class="l" name="262" href="#262">262</a> } 271*eeb7e5b3SAdam Hornáček<a class="l" name="263" href="#263">263</a> } 272*eeb7e5b3SAdam Hornáček<a class="l" name="264" href="#264">264</a> 273*eeb7e5b3SAdam Hornáček<a class="l" name="265" href="#265">265</a>} 274*eeb7e5b3SAdam Hornáček<a class="l" name="266" href="#266">266</a> 275*eeb7e5b3SAdam Hornáček<a class="l" name="267" href="#267">267</a> 276*eeb7e5b3SAdam Hornáček<a class="l" name="268" href="#268">268</a><b>private</b> <b>object</b> <a class="d" name="TorrentBroadcast"/><a href="/source/s?refs=TorrentBroadcast" class="d intelliWindow-symbol" data-definition-place="def">TorrentBroadcast</a> <b>extends</b> <a href="/source/s?defs=Logging" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Logging</a> { 277*eeb7e5b3SAdam Hornáček<a class="l" name="269" href="#269">269</a> 278*eeb7e5b3SAdam Hornáček<a class="hl" name="270" href="#270">270</a> <b>def</b> <a class="xmt" name="blockifyObject"/><a href="/source/s?refs=blockifyObject" class="xmt intelliWindow-symbol" data-definition-place="def">blockifyObject</a>[<a href="/source/s?defs=T" class="intelliWindow-symbol" data-definition-place="undefined-in-file">T</a>: <a href="/source/s?defs=ClassTag" class="intelliWindow-symbol" data-definition-place="undefined-in-file">ClassTag</a>]( 279*eeb7e5b3SAdam Hornáček<a class="l" name="271" href="#271">271</a> <a href="/source/s?defs=obj" class="intelliWindow-symbol" data-definition-place="undefined-in-file">obj</a>: <a href="/source/s?defs=T" class="intelliWindow-symbol" data-definition-place="undefined-in-file">T</a>, 280*eeb7e5b3SAdam Hornáček<a class="l" name="272" href="#272">272</a> <a href="/source/s?defs=blockSize" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blockSize</a>: <a href="/source/s?defs=Int" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Int</a>, 281*eeb7e5b3SAdam Hornáček<a class="l" name="273" href="#273">273</a> <a href="/source/s?defs=serializer" class="intelliWindow-symbol" data-definition-place="undefined-in-file">serializer</a>: <a href="/source/s?defs=Serializer" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Serializer</a>, 282*eeb7e5b3SAdam Hornáček<a class="l" name="274" href="#274">274</a> <a href="/source/s?defs=compressionCodec" class="intelliWindow-symbol" data-definition-place="undefined-in-file">compressionCodec</a>: <a href="/source/s?defs=Option" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Option</a>[<a href="/source/s?defs=CompressionCodec" class="intelliWindow-symbol" data-definition-place="undefined-in-file">CompressionCodec</a>]): <a href="/source/s?defs=Array" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Array</a>[<a href="/source/s?defs=ByteBuffer" class="intelliWindow-symbol" data-definition-place="undefined-in-file">ByteBuffer</a>] = { 283*eeb7e5b3SAdam Hornáček<a class="l" name="275" href="#275">275</a> <b>val</b> <a class="d" name="cbbos"/><a href="/source/s?refs=cbbos" class="d intelliWindow-symbol" data-definition-place="def">cbbos</a> = <b>new</b> <a href="/source/s?defs=ChunkedByteBufferOutputStream" class="intelliWindow-symbol" data-definition-place="undefined-in-file">ChunkedByteBufferOutputStream</a>(<a href="/source/s?defs=blockSize" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blockSize</a>, <a href="/source/s?defs=ByteBuffer" class="intelliWindow-symbol" data-definition-place="undefined-in-file">ByteBuffer</a>.<a href="/source/s?defs=allocate" class="intelliWindow-symbol" data-definition-place="undefined-in-file">allocate</a>) 284*eeb7e5b3SAdam Hornáček<a class="l" name="276" href="#276">276</a> <b>val</b> <a class="d" name="out"/><a href="/source/s?refs=out" class="d intelliWindow-symbol" data-definition-place="def">out</a> = <a href="/source/s?defs=compressionCodec" class="intelliWindow-symbol" data-definition-place="undefined-in-file">compressionCodec</a>.<a href="/source/s?defs=map" class="intelliWindow-symbol" data-definition-place="undefined-in-file">map</a>(<a href="/source/s?defs=c" class="intelliWindow-symbol" data-definition-place="undefined-in-file">c</a> => <a href="/source/s?defs=c" class="intelliWindow-symbol" data-definition-place="undefined-in-file">c</a>.<a href="/source/s?defs=compressedOutputStream" class="intelliWindow-symbol" data-definition-place="undefined-in-file">compressedOutputStream</a>(<a class="d intelliWindow-symbol" href="#cbbos" data-definition-place="defined-in-file">cbbos</a>)).<a href="/source/s?defs=getOrElse" class="intelliWindow-symbol" data-definition-place="undefined-in-file">getOrElse</a>(<a class="d intelliWindow-symbol" href="#cbbos" data-definition-place="defined-in-file">cbbos</a>) 285*eeb7e5b3SAdam Hornáček<a class="l" name="277" href="#277">277</a> <b>val</b> <a class="d" name="ser"/><a href="/source/s?refs=ser" class="d intelliWindow-symbol" data-definition-place="def">ser</a> = <a href="/source/s?defs=serializer" class="intelliWindow-symbol" data-definition-place="undefined-in-file">serializer</a>.<a href="/source/s?defs=newInstance" class="intelliWindow-symbol" data-definition-place="undefined-in-file">newInstance</a>() 286*eeb7e5b3SAdam Hornáček<a class="l" name="278" href="#278">278</a> <b>val</b> <a class="d" name="serOut"/><a href="/source/s?refs=serOut" class="d intelliWindow-symbol" data-definition-place="def">serOut</a> = <a href="/source/s?defs=ser" class="intelliWindow-symbol" data-definition-place="undefined-in-file">ser</a>.<a href="/source/s?defs=serializeStream" class="intelliWindow-symbol" data-definition-place="undefined-in-file">serializeStream</a>(<a class="d intelliWindow-symbol" href="#out" data-definition-place="defined-in-file">out</a>) 287*eeb7e5b3SAdam Hornáček<a class="l" name="279" href="#279">279</a> <a href="/source/s?defs=Utils" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Utils</a>.<a href="/source/s?defs=tryWithSafeFinally" class="intelliWindow-symbol" data-definition-place="undefined-in-file">tryWithSafeFinally</a> { 288*eeb7e5b3SAdam Hornáček<a class="hl" name="280" href="#280">280</a> <a class="d intelliWindow-symbol" href="#serOut" data-definition-place="defined-in-file">serOut</a>.<a href="/source/s?defs=writeObject" class="intelliWindow-symbol" data-definition-place="undefined-in-file">writeObject</a>[<a href="/source/s?defs=T" class="intelliWindow-symbol" data-definition-place="undefined-in-file">T</a>](<a href="/source/s?defs=obj" class="intelliWindow-symbol" data-definition-place="undefined-in-file">obj</a>) 289*eeb7e5b3SAdam Hornáček<a class="l" name="281" href="#281">281</a> } { 290*eeb7e5b3SAdam Hornáček<a class="l" name="282" href="#282">282</a> <a class="d intelliWindow-symbol" href="#serOut" data-definition-place="defined-in-file">serOut</a>.<a href="/source/s?defs=close" class="intelliWindow-symbol" data-definition-place="undefined-in-file">close</a>() 291*eeb7e5b3SAdam Hornáček<a class="l" name="283" href="#283">283</a> } 292*eeb7e5b3SAdam Hornáček<a class="l" name="284" href="#284">284</a> <a class="d intelliWindow-symbol" href="#cbbos" data-definition-place="defined-in-file">cbbos</a>.<a href="/source/s?defs=toChunkedByteBuffer" class="intelliWindow-symbol" data-definition-place="undefined-in-file">toChunkedByteBuffer</a>.<a href="/source/s?defs=getChunks" class="intelliWindow-symbol" data-definition-place="undefined-in-file">getChunks</a>() 293*eeb7e5b3SAdam Hornáček<a class="l" name="285" href="#285">285</a> } 294*eeb7e5b3SAdam Hornáček<a class="l" name="286" href="#286">286</a> 295*eeb7e5b3SAdam Hornáček<a class="l" name="287" href="#287">287</a> <b>def</b> <a class="xmt" name="unBlockifyObject"/><a href="/source/s?refs=unBlockifyObject" class="xmt intelliWindow-symbol" data-definition-place="def">unBlockifyObject</a>[<a href="/source/s?defs=T" class="intelliWindow-symbol" data-definition-place="undefined-in-file">T</a>: <a href="/source/s?defs=ClassTag" class="intelliWindow-symbol" data-definition-place="undefined-in-file">ClassTag</a>]( 296*eeb7e5b3SAdam Hornáček<a class="l" name="288" href="#288">288</a> <a href="/source/s?defs=blocks" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blocks</a>: <a href="/source/s?defs=Array" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Array</a>[<a href="/source/s?defs=InputStream" class="intelliWindow-symbol" data-definition-place="undefined-in-file">InputStream</a>], 297*eeb7e5b3SAdam Hornáček<a class="l" name="289" href="#289">289</a> <a href="/source/s?defs=serializer" class="intelliWindow-symbol" data-definition-place="undefined-in-file">serializer</a>: <a href="/source/s?defs=Serializer" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Serializer</a>, 298*eeb7e5b3SAdam Hornáček<a class="hl" name="290" href="#290">290</a> <a href="/source/s?defs=compressionCodec" class="intelliWindow-symbol" data-definition-place="undefined-in-file">compressionCodec</a>: <a href="/source/s?defs=Option" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Option</a>[<a href="/source/s?defs=CompressionCodec" class="intelliWindow-symbol" data-definition-place="undefined-in-file">CompressionCodec</a>]): <a href="/source/s?defs=T" class="intelliWindow-symbol" data-definition-place="undefined-in-file">T</a> = { 299*eeb7e5b3SAdam Hornáček<a class="l" name="291" href="#291">291</a> <a href="/source/s?defs=require" class="intelliWindow-symbol" data-definition-place="undefined-in-file">require</a>(<a href="/source/s?defs=blocks" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blocks</a>.<a href="/source/s?defs=nonEmpty" class="intelliWindow-symbol" data-definition-place="undefined-in-file">nonEmpty</a>, <span class="s">"Cannot unblockify an empty array of blocks"</span>) 300*eeb7e5b3SAdam Hornáček<a class="l" name="292" href="#292">292</a> <b>val</b> <a class="d" name="is"/><a href="/source/s?refs=is" class="d intelliWindow-symbol" data-definition-place="def">is</a> = <b>new</b> <a href="/source/s?defs=SequenceInputStream" class="intelliWindow-symbol" data-definition-place="undefined-in-file">SequenceInputStream</a>(<a href="/source/s?defs=blocks" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blocks</a>.<a href="/source/s?defs=iterator" class="intelliWindow-symbol" data-definition-place="undefined-in-file">iterator</a>.<a href="/source/s?defs=asJavaEnumeration" class="intelliWindow-symbol" data-definition-place="undefined-in-file">asJavaEnumeration</a>) 301*eeb7e5b3SAdam Hornáček<a class="l" name="293" href="#293">293</a> <b>val</b> <a class="d" name="in"/><a href="/source/s?refs=in" class="d intelliWindow-symbol" data-definition-place="def">in</a>: <a href="/source/s?defs=InputStream" class="intelliWindow-symbol" data-definition-place="undefined-in-file">InputStream</a> = <a href="/source/s?defs=compressionCodec" class="intelliWindow-symbol" data-definition-place="undefined-in-file">compressionCodec</a>.<a href="/source/s?defs=map" class="intelliWindow-symbol" data-definition-place="undefined-in-file">map</a>(<a href="/source/s?defs=c" class="intelliWindow-symbol" data-definition-place="undefined-in-file">c</a> => <a href="/source/s?defs=c" class="intelliWindow-symbol" data-definition-place="undefined-in-file">c</a>.<a href="/source/s?defs=compressedInputStream" class="intelliWindow-symbol" data-definition-place="undefined-in-file">compressedInputStream</a>(<a class="d intelliWindow-symbol" href="#is" data-definition-place="defined-in-file">is</a>)).<a href="/source/s?defs=getOrElse" class="intelliWindow-symbol" data-definition-place="undefined-in-file">getOrElse</a>(<a class="d intelliWindow-symbol" href="#is" data-definition-place="defined-in-file">is</a>) 302*eeb7e5b3SAdam Hornáček<a class="l" name="294" href="#294">294</a> <b>val</b> <a class="d" name="ser"/><a href="/source/s?refs=ser" class="d intelliWindow-symbol" data-definition-place="def">ser</a> = <a href="/source/s?defs=serializer" class="intelliWindow-symbol" data-definition-place="undefined-in-file">serializer</a>.<a href="/source/s?defs=newInstance" class="intelliWindow-symbol" data-definition-place="undefined-in-file">newInstance</a>() 303*eeb7e5b3SAdam Hornáček<a class="l" name="295" href="#295">295</a> <b>val</b> <a class="d" name="serIn"/><a href="/source/s?refs=serIn" class="d intelliWindow-symbol" data-definition-place="def">serIn</a> = <a href="/source/s?defs=ser" class="intelliWindow-symbol" data-definition-place="undefined-in-file">ser</a>.<a href="/source/s?defs=deserializeStream" class="intelliWindow-symbol" data-definition-place="undefined-in-file">deserializeStream</a>(<a class="d intelliWindow-symbol" href="#in" data-definition-place="defined-in-file">in</a>) 304*eeb7e5b3SAdam Hornáček<a class="l" name="296" href="#296">296</a> <b>val</b> <a class="d" name="obj"/><a href="/source/s?refs=obj" class="d intelliWindow-symbol" data-definition-place="def">obj</a> = <a href="/source/s?defs=Utils" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Utils</a>.<a href="/source/s?defs=tryWithSafeFinally" class="intelliWindow-symbol" data-definition-place="undefined-in-file">tryWithSafeFinally</a> { 305*eeb7e5b3SAdam Hornáček<a class="l" name="297" href="#297">297</a> <a class="d intelliWindow-symbol" href="#serIn" data-definition-place="defined-in-file">serIn</a>.<a href="/source/s?defs=readObject" class="intelliWindow-symbol" data-definition-place="undefined-in-file">readObject</a>[<a href="/source/s?defs=T" class="intelliWindow-symbol" data-definition-place="undefined-in-file">T</a>]() 306*eeb7e5b3SAdam Hornáček<a class="l" name="298" href="#298">298</a> } { 307*eeb7e5b3SAdam Hornáček<a class="l" name="299" href="#299">299</a> <a class="d intelliWindow-symbol" href="#serIn" data-definition-place="defined-in-file">serIn</a>.`<a href="/source/s?defs=close" class="intelliWindow-symbol" data-definition-place="undefined-in-file">close</a>`() 308*eeb7e5b3SAdam Hornáček<a class="hl" name="300" href="#300">300</a> } 309*eeb7e5b3SAdam Hornáček<a class="l" name="301" href="#301">301</a> <a href="/source/s?defs=obj" class="intelliWindow-symbol" data-definition-place="undefined-in-file">obj</a> 310*eeb7e5b3SAdam Hornáček<a class="l" name="302" href="#302">302</a> } 311*eeb7e5b3SAdam Hornáček<a class="l" name="303" href="#303">303</a> 312*eeb7e5b3SAdam Hornáček<a class="l" name="304" href="#304">304</a> <span class="c">/**</span> 313*eeb7e5b3SAdam Hornáček<a class="l" name="305" href="#305">305</a><span class="c"> * Remove all persisted blocks associated with this torrent broadcast on the executors.</span> 314*eeb7e5b3SAdam Hornáček<a class="l" name="306" href="#306">306</a><span class="c"> * If removeFromDriver is true, also remove these persisted blocks on the driver.</span> 315*eeb7e5b3SAdam Hornáček<a class="l" name="307" href="#307">307</a><span class="c"> */</span> 316*eeb7e5b3SAdam Hornáček<a class="l" name="308" href="#308">308</a> <b>def</b> <a class="xmt" name="unpersist"/><a href="/source/s?refs=unpersist" class="xmt intelliWindow-symbol" data-definition-place="def">unpersist</a>(<a href="/source/s?defs=id" class="intelliWindow-symbol" data-definition-place="undefined-in-file">id</a>: <a href="/source/s?defs=Long" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Long</a>, <a href="/source/s?defs=removeFromDriver" class="intelliWindow-symbol" data-definition-place="undefined-in-file">removeFromDriver</a>: <a href="/source/s?defs=Boolean" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Boolean</a>, <a href="/source/s?defs=blocking" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blocking</a>: <a href="/source/s?defs=Boolean" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Boolean</a>): <a href="/source/s?defs=Unit" class="intelliWindow-symbol" data-definition-place="undefined-in-file">Unit</a> = { 317*eeb7e5b3SAdam Hornáček<a class="l" name="309" href="#309">309</a> <a href="/source/s?defs=logDebug" class="intelliWindow-symbol" data-definition-place="undefined-in-file">logDebug</a>(s<span class="s">"<a href="http://example.com?">http://example.com?</a>$</span><a href="/source/s?defs=id" class="intelliWindow-symbol" data-definition-place="undefined-in-file">id</a><span class="s">"</span>) 318*eeb7e5b3SAdam Hornáček<a class="hl" name="310" href="#310">310</a> <a href="/source/s?defs=SparkEnv" class="intelliWindow-symbol" data-definition-place="undefined-in-file">SparkEnv</a>.<a href="/source/s?defs=get" class="intelliWindow-symbol" data-definition-place="undefined-in-file">get</a>.<a href="/source/s?defs=blockManager" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blockManager</a>.<a href="/source/s?defs=master" class="intelliWindow-symbol" data-definition-place="undefined-in-file">master</a>.<a href="/source/s?defs=removeBroadcast" class="intelliWindow-symbol" data-definition-place="undefined-in-file">removeBroadcast</a>(<a href="/source/s?defs=id" class="intelliWindow-symbol" data-definition-place="undefined-in-file">id</a>, <a href="/source/s?defs=removeFromDriver" class="intelliWindow-symbol" data-definition-place="undefined-in-file">removeFromDriver</a>, <a href="/source/s?defs=blocking" class="intelliWindow-symbol" data-definition-place="undefined-in-file">blocking</a>) 319*eeb7e5b3SAdam Hornáček<a class="l" name="311" href="#311">311</a> } 320*eeb7e5b3SAdam Hornáček<a class="l" name="312" href="#312">312</a> 321*eeb7e5b3SAdam Hornáček<a class="l" name="313" href="#313">313</a> <span class="c">/*<a href="http://example.com">http://example.com</a>.*/</span> 322*eeb7e5b3SAdam Hornáček<a class="l" name="314" href="#314">314</a> <span class="c">/* comment /* comment */</span> 323*eeb7e5b3SAdam Hornáček<a class="l" name="315" href="#315">315</a><span class="c"> comment</span> 324*eeb7e5b3SAdam Hornáček<a class="l" name="316" href="#316">316</a><span class="c"> */</span> 325*eeb7e5b3SAdam Hornáček<a class="l" name="317" href="#317">317</a> 326*eeb7e5b3SAdam Hornáček<a class="l" name="318" href="#318">318</a> <b>def</b> <a class="xmt" name="unary_"/><a href="/source/s?refs=unary_" class="xmt intelliWindow-symbol" data-definition-place="def">unary_</a>~ = <span class="n">0</span> 327*eeb7e5b3SAdam Hornáček<a class="l" name="319" href="#319">319</a> <span class="c">/***/</span> 328*eeb7e5b3SAdam Hornáček<a class="hl" name="320" href="#320">320</a>} 329*eeb7e5b3SAdam Hornáček<a class="l" name="321" href="#321">321</a></body> 330*eeb7e5b3SAdam Hornáček</html> 331